2024-12-09 08:17:45,757 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b 2024-12-09 08:17:45,772 main DEBUG Took 0.012409 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-09 08:17:45,772 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-09 08:17:45,773 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-09 08:17:45,774 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-09 08:17:45,775 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:17:45,786 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-09 08:17:45,802 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:17:45,803 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:17:45,804 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:17:45,805 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:17:45,806 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:17:45,806 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:17:45,807 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:17:45,808 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:17:45,809 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:17:45,809 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:17:45,810 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:17:45,811 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:17:45,812 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:17:45,812 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:17:45,813 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:17:45,813 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:17:45,814 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:17:45,814 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:17:45,815 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:17:45,815 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:17:45,816 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:17:45,816 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:17:45,817 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:17:45,818 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:17:45,819 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:17:45,819 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-09 08:17:45,821 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:17:45,822 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-09 08:17:45,824 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-09 08:17:45,824 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-09 08:17:45,826 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-09 08:17:45,826 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-09 08:17:45,834 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-09 08:17:45,837 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-09 08:17:45,838 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-09 08:17:45,839 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-09 08:17:45,839 main DEBUG createAppenders(={Console}) 2024-12-09 08:17:45,840 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b initialized 2024-12-09 08:17:45,840 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b 2024-12-09 08:17:45,841 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b OK. 2024-12-09 08:17:45,841 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-09 08:17:45,841 main DEBUG OutputStream closed 2024-12-09 08:17:45,842 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-09 08:17:45,842 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-09 08:17:45,842 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@37d4349f OK 2024-12-09 08:17:45,910 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-09 08:17:45,912 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-09 08:17:45,913 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-09 08:17:45,914 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-09 08:17:45,914 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-09 08:17:45,915 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-09 08:17:45,915 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-09 08:17:45,915 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-09 08:17:45,915 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-09 08:17:45,916 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-09 08:17:45,916 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-09 08:17:45,916 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-09 08:17:45,917 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-09 08:17:45,917 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-09 08:17:45,917 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-09 08:17:45,917 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-09 08:17:45,918 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-09 08:17:45,918 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-09 08:17:45,920 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09 08:17:45,920 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4331d187) with optional ClassLoader: null 2024-12-09 08:17:45,921 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-09 08:17:45,921 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4331d187] started OK. 2024-12-09T08:17:45,933 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync timeout: 13 mins 2024-12-09 08:17:45,935 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-09 08:17:45,935 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09T08:17:45,992 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.TestFailedAppendAndSync#testLockupAroundBadAssignSync Thread=11, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=353, ProcessCount=11, AvailableMemoryMB=5674 2024-12-09T08:17:46,242 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0 2024-12-09T08:17:46,736 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-09T08:17:46,823 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-09T08:17:46,881 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=wal, suffix=, logDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync, archiveDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/oldWALs, maxLogs=55 2024-12-09T08:17:46,922 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732266914 2024-12-09T08:17:46,996 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732266914 2024-12-09T08:17:47,055 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,061 INFO [Time-limited test {}] regionserver.ChunkCreator(472): data poolSizePercentage is less than 0. So not using pool 2024-12-09T08:17:47,061 INFO [Time-limited test {}] regionserver.ChunkCreator(472): index poolSizePercentage is less than 0. So not using pool 2024-12-09T08:17:47,095 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 9272fedbd8f9b1f79bbb491a798c861c, NAME => 'testLockupAroundBadAssignSync,,1733732267074.9272fedbd8f9b1f79bbb491a798c861c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testLockupAroundBadAssignSync', {TABLE_ATTRIBUTES => {DURABILITY => 'SYNC_WAL', READONLY => 'false'}}, {NAME => 'MyCF', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)', METADATA => {'IN_MEMORY_COMPACTION' => 'NONE'}}, regionDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0 2024-12-09T08:17:47,133 INFO [Time-limited test {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-09T08:17:47,137 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testLockupAroundBadAssignSync,,1733732267074.9272fedbd8f9b1f79bbb491a798c861c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:17:47,216 INFO [StoreOpener-9272fedbd8f9b1f79bbb491a798c861c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family MyCF of region 9272fedbd8f9b1f79bbb491a798c861c 2024-12-09T08:17:47,248 INFO [StoreOpener-9272fedbd8f9b1f79bbb491a798c861c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9272fedbd8f9b1f79bbb491a798c861c columnFamilyName MyCF 2024-12-09T08:17:47,255 DEBUG [StoreOpener-9272fedbd8f9b1f79bbb491a798c861c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:17:47,262 INFO [StoreOpener-9272fedbd8f9b1f79bbb491a798c861c-1 {}] regionserver.HStore(327): Store=9272fedbd8f9b1f79bbb491a798c861c/MyCF, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:17:47,266 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9272fedbd8f9b1f79bbb491a798c861c 2024-12-09T08:17:47,272 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/data/default/testLockupAroundBadAssignSync/9272fedbd8f9b1f79bbb491a798c861c 2024-12-09T08:17:47,273 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/data/default/testLockupAroundBadAssignSync/9272fedbd8f9b1f79bbb491a798c861c 2024-12-09T08:17:47,275 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9272fedbd8f9b1f79bbb491a798c861c 2024-12-09T08:17:47,276 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9272fedbd8f9b1f79bbb491a798c861c 2024-12-09T08:17:47,280 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9272fedbd8f9b1f79bbb491a798c861c 2024-12-09T08:17:47,289 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/data/default/testLockupAroundBadAssignSync/9272fedbd8f9b1f79bbb491a798c861c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:17:47,290 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9272fedbd8f9b1f79bbb491a798c861c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60617657, jitterRate=-0.09672652184963226}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T08:17:47,298 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-09T08:17:47,310 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9272fedbd8f9b1f79bbb491a798c861c: Writing region info on filesystem at 1733732267184Initializing all the Stores at 1733732267186 (+2 ms)Instantiating store for column family {NAME => 'MyCF', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)', METADATA => {'IN_MEMORY_COMPACTION' => 'NONE'}} at 1733732267186Cleaning up temporary data from old regions at 1733732267276 (+90 ms)Region opened successfully at 1733732267308 (+32 ms) 2024-12-09T08:17:47,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732266914) roll requested 2024-12-09T08:17:47,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267356 2024-12-09T08:17:47,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732266914 with entries=1, filesize=249 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267356 2024-12-09T08:17:47,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267356) roll requested 2024-12-09T08:17:47,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267370 2024-12-09T08:17:47,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267370 2024-12-09T08:17:47,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267370) roll requested 2024-12-09T08:17:47,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267384 2024-12-09T08:17:47,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267384 2024-12-09T08:17:47,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267384) roll requested 2024-12-09T08:17:47,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267400 2024-12-09T08:17:47,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267400 2024-12-09T08:17:47,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,409 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267400) roll requested 2024-12-09T08:17:47,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267410 2024-12-09T08:17:47,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267410 2024-12-09T08:17:47,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267410) roll requested 2024-12-09T08:17:47,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267421 2024-12-09T08:17:47,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267410 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267421 2024-12-09T08:17:47,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,429 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267421) roll requested 2024-12-09T08:17:47,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267430 2024-12-09T08:17:47,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267430 2024-12-09T08:17:47,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267430) roll requested 2024-12-09T08:17:47,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267440 2024-12-09T08:17:47,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267440 2024-12-09T08:17:47,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,451 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267440) roll requested 2024-12-09T08:17:47,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267452 2024-12-09T08:17:47,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267452 2024-12-09T08:17:47,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267452) roll requested 2024-12-09T08:17:47,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267462 2024-12-09T08:17:47,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267462 2024-12-09T08:17:47,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267462) roll requested 2024-12-09T08:17:47,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267472 2024-12-09T08:17:47,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267462 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267472 2024-12-09T08:17:47,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267472) roll requested 2024-12-09T08:17:47,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267481 2024-12-09T08:17:47,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267481 2024-12-09T08:17:47,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267481) roll requested 2024-12-09T08:17:47,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267489 2024-12-09T08:17:47,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267489 2024-12-09T08:17:47,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,498 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267489) roll requested 2024-12-09T08:17:47,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267499 2024-12-09T08:17:47,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267499 2024-12-09T08:17:47,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,509 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267499) roll requested 2024-12-09T08:17:47,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267510 2024-12-09T08:17:47,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267499 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267510 2024-12-09T08:17:47,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267510) roll requested 2024-12-09T08:17:47,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267520 2024-12-09T08:17:47,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267510 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267520 2024-12-09T08:17:47,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,529 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267520) roll requested 2024-12-09T08:17:47,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267530 2024-12-09T08:17:47,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267530 2024-12-09T08:17:47,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267530) roll requested 2024-12-09T08:17:47,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267541 2024-12-09T08:17:47,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267541 2024-12-09T08:17:47,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,551 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267541) roll requested 2024-12-09T08:17:47,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267552 2024-12-09T08:17:47,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267552 2024-12-09T08:17:47,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267552) roll requested 2024-12-09T08:17:47,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267561 2024-12-09T08:17:47,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267561 2024-12-09T08:17:47,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267561) roll requested 2024-12-09T08:17:47,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267572 2024-12-09T08:17:47,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267572 2024-12-09T08:17:47,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,586 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267572) roll requested 2024-12-09T08:17:47,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267587 2024-12-09T08:17:47,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267587 2024-12-09T08:17:47,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267587) roll requested 2024-12-09T08:17:47,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267596 2024-12-09T08:17:47,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267596 2024-12-09T08:17:47,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267596) roll requested 2024-12-09T08:17:47,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267604 2024-12-09T08:17:47,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267596 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267604 2024-12-09T08:17:47,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267604) roll requested 2024-12-09T08:17:47,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267612 2024-12-09T08:17:47,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267612 2024-12-09T08:17:47,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,619 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267612) roll requested 2024-12-09T08:17:47,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267619 2024-12-09T08:17:47,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267619 2024-12-09T08:17:47,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267619) roll requested 2024-12-09T08:17:47,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267627 2024-12-09T08:17:47,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267627 2024-12-09T08:17:47,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,635 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267627) roll requested 2024-12-09T08:17:47,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267636 2024-12-09T08:17:47,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267627 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267636 2024-12-09T08:17:47,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267636) roll requested 2024-12-09T08:17:47,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267643 2024-12-09T08:17:47,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267636 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267643 2024-12-09T08:17:47,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267643) roll requested 2024-12-09T08:17:47,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267652 2024-12-09T08:17:47,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267643 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267652 2024-12-09T08:17:47,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267652) roll requested 2024-12-09T08:17:47,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267660 2024-12-09T08:17:47,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267652 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267660 2024-12-09T08:17:47,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267660) roll requested 2024-12-09T08:17:47,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267669 2024-12-09T08:17:47,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267669 2024-12-09T08:17:47,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,679 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267669) roll requested 2024-12-09T08:17:47,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267679 2024-12-09T08:17:47,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267679 2024-12-09T08:17:47,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,687 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267679) roll requested 2024-12-09T08:17:47,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267687 2024-12-09T08:17:47,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267679 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267687 2024-12-09T08:17:47,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267687) roll requested 2024-12-09T08:17:47,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267696 2024-12-09T08:17:47,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267687 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267696 2024-12-09T08:17:47,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267696) roll requested 2024-12-09T08:17:47,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267704 2024-12-09T08:17:47,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267696 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267704 2024-12-09T08:17:47,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267704) roll requested 2024-12-09T08:17:47,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267713 2024-12-09T08:17:47,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267713 2024-12-09T08:17:47,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,722 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267713) roll requested 2024-12-09T08:17:47,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267722 2024-12-09T08:17:47,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267722 2024-12-09T08:17:47,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267722) roll requested 2024-12-09T08:17:47,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267732 2024-12-09T08:17:47,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267722 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267732 2024-12-09T08:17:47,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,743 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267732) roll requested 2024-12-09T08:17:47,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267744 2024-12-09T08:17:47,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267744 2024-12-09T08:17:47,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267744) roll requested 2024-12-09T08:17:47,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267751 2024-12-09T08:17:47,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267751 2024-12-09T08:17:47,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,758 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267751) roll requested 2024-12-09T08:17:47,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267759 2024-12-09T08:17:47,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267759 2024-12-09T08:17:47,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267759) roll requested 2024-12-09T08:17:47,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267766 2024-12-09T08:17:47,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267766 2024-12-09T08:17:47,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,774 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267766) roll requested 2024-12-09T08:17:47,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267775 2024-12-09T08:17:47,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267766 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267775 2024-12-09T08:17:47,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267775) roll requested 2024-12-09T08:17:47,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267786 2024-12-09T08:17:47,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267786 2024-12-09T08:17:47,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,792 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267786) roll requested 2024-12-09T08:17:47,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267793 2024-12-09T08:17:47,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267786 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267793 2024-12-09T08:17:47,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267793) roll requested 2024-12-09T08:17:47,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267800 2024-12-09T08:17:47,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267800 2024-12-09T08:17:47,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,807 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267800) roll requested 2024-12-09T08:17:47,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267808 2024-12-09T08:17:47,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267808 2024-12-09T08:17:47,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,817 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267808) roll requested 2024-12-09T08:17:47,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267818 2024-12-09T08:17:47,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267818 2024-12-09T08:17:47,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267818) roll requested 2024-12-09T08:17:47,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267828 2024-12-09T08:17:47,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267828 2024-12-09T08:17:47,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,838 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267828) roll requested 2024-12-09T08:17:47,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267839 2024-12-09T08:17:47,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267828 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267839 2024-12-09T08:17:47,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267839) roll requested 2024-12-09T08:17:47,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267849 2024-12-09T08:17:47,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267839 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267849 2024-12-09T08:17:47,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267849) roll requested 2024-12-09T08:17:47,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267857 2024-12-09T08:17:47,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267857 2024-12-09T08:17:47,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,863 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267857) roll requested 2024-12-09T08:17:47,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267863 2024-12-09T08:17:47,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267863 2024-12-09T08:17:47,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,874 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267863) roll requested 2024-12-09T08:17:47,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267874 2024-12-09T08:17:47,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267863 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267874 2024-12-09T08:17:47,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267874) roll requested 2024-12-09T08:17:47,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267881 2024-12-09T08:17:47,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267881 2024-12-09T08:17:47,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,889 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267881) roll requested 2024-12-09T08:17:47,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267889 2024-12-09T08:17:47,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267889 2024-12-09T08:17:47,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267889) roll requested 2024-12-09T08:17:47,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267896 2024-12-09T08:17:47,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267896 2024-12-09T08:17:47,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,903 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267896) roll requested 2024-12-09T08:17:47,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267904 2024-12-09T08:17:47,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267904 2024-12-09T08:17:47,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,911 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267904) roll requested 2024-12-09T08:17:47,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267912 2024-12-09T08:17:47,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267904 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267912 2024-12-09T08:17:47,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267912) roll requested 2024-12-09T08:17:47,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267921 2024-12-09T08:17:47,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267921 2024-12-09T08:17:47,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,929 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267912 is not closed yet, will try archiving it next time 2024-12-09T08:17:47,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267921) roll requested 2024-12-09T08:17:47,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267930 2024-12-09T08:17:47,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267930 2024-12-09T08:17:47,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267930) roll requested 2024-12-09T08:17:47,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267942 2024-12-09T08:17:47,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267930 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267942 2024-12-09T08:17:47,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267942) roll requested 2024-12-09T08:17:47,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267952 2024-12-09T08:17:47,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267952 2024-12-09T08:17:47,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267952) roll requested 2024-12-09T08:17:47,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267967 2024-12-09T08:17:47,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267967 2024-12-09T08:17:47,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,978 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267967) roll requested 2024-12-09T08:17:47,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267978 2024-12-09T08:17:47,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267967 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267978 2024-12-09T08:17:47,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,984 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267978) roll requested 2024-12-09T08:17:47,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267985 2024-12-09T08:17:47,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267978 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267985 2024-12-09T08:17:47,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267985) roll requested 2024-12-09T08:17:47,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267991 2024-12-09T08:17:47,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:47,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267991 2024-12-09T08:17:47,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:47,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:47,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267991) roll requested 2024-12-09T08:17:47,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732267998 2024-12-09T08:17:48,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267998 2024-12-09T08:17:48,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732267998) roll requested 2024-12-09T08:17:48,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268004 2024-12-09T08:17:48,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732267998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268004 2024-12-09T08:17:48,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268004) roll requested 2024-12-09T08:17:48,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268010 2024-12-09T08:17:48,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268010 2024-12-09T08:17:48,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,015 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268010) roll requested 2024-12-09T08:17:48,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268016 2024-12-09T08:17:48,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268016 2024-12-09T08:17:48,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268016) roll requested 2024-12-09T08:17:48,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268023 2024-12-09T08:17:48,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268023 2024-12-09T08:17:48,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,031 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268023) roll requested 2024-12-09T08:17:48,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268032 2024-12-09T08:17:48,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268032 2024-12-09T08:17:48,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268032) roll requested 2024-12-09T08:17:48,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268041 2024-12-09T08:17:48,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268041 2024-12-09T08:17:48,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268041) roll requested 2024-12-09T08:17:48,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268050 2024-12-09T08:17:48,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268050 2024-12-09T08:17:48,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,058 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268050) roll requested 2024-12-09T08:17:48,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268058 2024-12-09T08:17:48,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268050 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268058 2024-12-09T08:17:48,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,066 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268058) roll requested 2024-12-09T08:17:48,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268067 2024-12-09T08:17:48,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268058 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268067 2024-12-09T08:17:48,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,075 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268067) roll requested 2024-12-09T08:17:48,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268076 2024-12-09T08:17:48,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268067 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268076 2024-12-09T08:17:48,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268076) roll requested 2024-12-09T08:17:48,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268083 2024-12-09T08:17:48,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268083 2024-12-09T08:17:48,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268083) roll requested 2024-12-09T08:17:48,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268093 2024-12-09T08:17:48,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268093 2024-12-09T08:17:48,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268093) roll requested 2024-12-09T08:17:48,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268103 2024-12-09T08:17:48,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268103 2024-12-09T08:17:48,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268103) roll requested 2024-12-09T08:17:48,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268109 2024-12-09T08:17:48,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268109 2024-12-09T08:17:48,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,117 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268109) roll requested 2024-12-09T08:17:48,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268117 2024-12-09T08:17:48,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268117 2024-12-09T08:17:48,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268117) roll requested 2024-12-09T08:17:48,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268130 2024-12-09T08:17:48,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268130 2024-12-09T08:17:48,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268130) roll requested 2024-12-09T08:17:48,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268136 2024-12-09T08:17:48,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268130 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268136 2024-12-09T08:17:48,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268136) roll requested 2024-12-09T08:17:48,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268143 2024-12-09T08:17:48,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268143 2024-12-09T08:17:48,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,149 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268143) roll requested 2024-12-09T08:17:48,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268150 2024-12-09T08:17:48,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268150 2024-12-09T08:17:48,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268150) roll requested 2024-12-09T08:17:48,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268161 2024-12-09T08:17:48,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268161 2024-12-09T08:17:48,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268161) roll requested 2024-12-09T08:17:48,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268171 2024-12-09T08:17:48,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268171 2024-12-09T08:17:48,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268171) roll requested 2024-12-09T08:17:48,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268180 2024-12-09T08:17:48,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268180 2024-12-09T08:17:48,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268180) roll requested 2024-12-09T08:17:48,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268189 2024-12-09T08:17:48,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268189 2024-12-09T08:17:48,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,198 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268189) roll requested 2024-12-09T08:17:48,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268199 2024-12-09T08:17:48,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268199 2024-12-09T08:17:48,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,207 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268199) roll requested 2024-12-09T08:17:48,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268208 2024-12-09T08:17:48,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268199 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268208 2024-12-09T08:17:48,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268208) roll requested 2024-12-09T08:17:48,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268220 2024-12-09T08:17:48,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268220 2024-12-09T08:17:48,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,235 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268220) roll requested 2024-12-09T08:17:48,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268235 2024-12-09T08:17:48,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268235 2024-12-09T08:17:48,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,241 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268235) roll requested 2024-12-09T08:17:48,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268241 2024-12-09T08:17:48,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268235 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268241 2024-12-09T08:17:48,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268241) roll requested 2024-12-09T08:17:48,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268250 2024-12-09T08:17:48,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268241 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268250 2024-12-09T08:17:48,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268250) roll requested 2024-12-09T08:17:48,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268257 2024-12-09T08:17:48,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268250 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268257 2024-12-09T08:17:48,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268257) roll requested 2024-12-09T08:17:48,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268264 2024-12-09T08:17:48,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268264 2024-12-09T08:17:48,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268264) roll requested 2024-12-09T08:17:48,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268271 2024-12-09T08:17:48,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268264 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268271 2024-12-09T08:17:48,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268271) roll requested 2024-12-09T08:17:48,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268278 2024-12-09T08:17:48,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268278 2024-12-09T08:17:48,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268278) roll requested 2024-12-09T08:17:48,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268285 2024-12-09T08:17:48,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268285 2024-12-09T08:17:48,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268285) roll requested 2024-12-09T08:17:48,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268293 2024-12-09T08:17:48,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268293 2024-12-09T08:17:48,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268293) roll requested 2024-12-09T08:17:48,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268302 2024-12-09T08:17:48,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268302 2024-12-09T08:17:48,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,309 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268302) roll requested 2024-12-09T08:17:48,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268310 2024-12-09T08:17:48,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268302 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268310 2024-12-09T08:17:48,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268310) roll requested 2024-12-09T08:17:48,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268317 2024-12-09T08:17:48,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268317 2024-12-09T08:17:48,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,325 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268317) roll requested 2024-12-09T08:17:48,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268326 2024-12-09T08:17:48,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268326 2024-12-09T08:17:48,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268326) roll requested 2024-12-09T08:17:48,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268332 2024-12-09T08:17:48,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268326 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268332 2024-12-09T08:17:48,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,342 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268332) roll requested 2024-12-09T08:17:48,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268342 2024-12-09T08:17:48,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268342 2024-12-09T08:17:48,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,349 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268342) roll requested 2024-12-09T08:17:48,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268349 2024-12-09T08:17:48,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268342 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268349 2024-12-09T08:17:48,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,355 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268349) roll requested 2024-12-09T08:17:48,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268355 2024-12-09T08:17:48,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268349 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268355 2024-12-09T08:17:48,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268355) roll requested 2024-12-09T08:17:48,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268363 2024-12-09T08:17:48,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268363 2024-12-09T08:17:48,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268363) roll requested 2024-12-09T08:17:48,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268372 2024-12-09T08:17:48,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268372 2024-12-09T08:17:48,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,380 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268372) roll requested 2024-12-09T08:17:48,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268380 2024-12-09T08:17:48,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268380 2024-12-09T08:17:48,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,388 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268380) roll requested 2024-12-09T08:17:48,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268388 2024-12-09T08:17:48,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268388 2024-12-09T08:17:48,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,395 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268388) roll requested 2024-12-09T08:17:48,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268396 2024-12-09T08:17:48,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268388 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268396 2024-12-09T08:17:48,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,402 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268396) roll requested 2024-12-09T08:17:48,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268402 2024-12-09T08:17:48,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,408 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268402 2024-12-09T08:17:48,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,409 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268402) roll requested 2024-12-09T08:17:48,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268409 2024-12-09T08:17:48,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268402 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268409 2024-12-09T08:17:48,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,416 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268409) roll requested 2024-12-09T08:17:48,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268417 2024-12-09T08:17:48,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,424 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268409 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268417 2024-12-09T08:17:48,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268417) roll requested 2024-12-09T08:17:48,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268425 2024-12-09T08:17:48,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268417 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268425 2024-12-09T08:17:48,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268425) roll requested 2024-12-09T08:17:48,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268435 2024-12-09T08:17:48,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268435 2024-12-09T08:17:48,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,444 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268435) roll requested 2024-12-09T08:17:48,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268445 2024-12-09T08:17:48,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268435 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268445 2024-12-09T08:17:48,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,454 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268445) roll requested 2024-12-09T08:17:48,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268455 2024-12-09T08:17:48,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268455 2024-12-09T08:17:48,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268455) roll requested 2024-12-09T08:17:48,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268465 2024-12-09T08:17:48,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268455 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268465 2024-12-09T08:17:48,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268465) roll requested 2024-12-09T08:17:48,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268473 2024-12-09T08:17:48,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268473 2024-12-09T08:17:48,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268473) roll requested 2024-12-09T08:17:48,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268479 2024-12-09T08:17:48,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268479 2024-12-09T08:17:48,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,485 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268479) roll requested 2024-12-09T08:17:48,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268486 2024-12-09T08:17:48,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268479 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268486 2024-12-09T08:17:48,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,493 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268486) roll requested 2024-12-09T08:17:48,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268493 2024-12-09T08:17:48,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268486 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268493 2024-12-09T08:17:48,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268493) roll requested 2024-12-09T08:17:48,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268503 2024-12-09T08:17:48,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268503 2024-12-09T08:17:48,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268503) roll requested 2024-12-09T08:17:48,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268519 2024-12-09T08:17:48,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268519 2024-12-09T08:17:48,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,525 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268519) roll requested 2024-12-09T08:17:48,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268525 2024-12-09T08:17:48,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268525 2024-12-09T08:17:48,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268525) roll requested 2024-12-09T08:17:48,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268531 2024-12-09T08:17:48,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268525 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268531 2024-12-09T08:17:48,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268531) roll requested 2024-12-09T08:17:48,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268537 2024-12-09T08:17:48,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268537 2024-12-09T08:17:48,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,542 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268537) roll requested 2024-12-09T08:17:48,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268543 2024-12-09T08:17:48,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268543 2024-12-09T08:17:48,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,551 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268543) roll requested 2024-12-09T08:17:48,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268551 2024-12-09T08:17:48,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268551 2024-12-09T08:17:48,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268551) roll requested 2024-12-09T08:17:48,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268557 2024-12-09T08:17:48,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268551 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268557 2024-12-09T08:17:48,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,562 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268557) roll requested 2024-12-09T08:17:48,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268563 2024-12-09T08:17:48,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268563 2024-12-09T08:17:48,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268563) roll requested 2024-12-09T08:17:48,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268570 2024-12-09T08:17:48,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268563 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268570 2024-12-09T08:17:48,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268570) roll requested 2024-12-09T08:17:48,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268578 2024-12-09T08:17:48,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268570 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268578 2024-12-09T08:17:48,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268578) roll requested 2024-12-09T08:17:48,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268586 2024-12-09T08:17:48,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268586 2024-12-09T08:17:48,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,594 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268586) roll requested 2024-12-09T08:17:48,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268595 2024-12-09T08:17:48,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268586 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268595 2024-12-09T08:17:48,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268595) roll requested 2024-12-09T08:17:48,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268603 2024-12-09T08:17:48,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268603 2024-12-09T08:17:48,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268603) roll requested 2024-12-09T08:17:48,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268612 2024-12-09T08:17:48,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268603 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268612 2024-12-09T08:17:48,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,618 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268612) roll requested 2024-12-09T08:17:48,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268619 2024-12-09T08:17:48,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268619 2024-12-09T08:17:48,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,641 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268619) roll requested 2024-12-09T08:17:48,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268641 2024-12-09T08:17:48,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268641 2024-12-09T08:17:48,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268641) roll requested 2024-12-09T08:17:48,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268653 2024-12-09T08:17:48,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268653 2024-12-09T08:17:48,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268653) roll requested 2024-12-09T08:17:48,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268659 2024-12-09T08:17:48,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268659 2024-12-09T08:17:48,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,665 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268659) roll requested 2024-12-09T08:17:48,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268665 2024-12-09T08:17:48,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268659 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268665 2024-12-09T08:17:48,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,671 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268665) roll requested 2024-12-09T08:17:48,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268671 2024-12-09T08:17:48,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268665 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268671 2024-12-09T08:17:48,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268671) roll requested 2024-12-09T08:17:48,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268678 2024-12-09T08:17:48,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268671 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268678 2024-12-09T08:17:48,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268678) roll requested 2024-12-09T08:17:48,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268685 2024-12-09T08:17:48,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268685 2024-12-09T08:17:48,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268685) roll requested 2024-12-09T08:17:48,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268691 2024-12-09T08:17:48,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268691 2024-12-09T08:17:48,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268691) roll requested 2024-12-09T08:17:48,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268696 2024-12-09T08:17:48,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268691 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268696 2024-12-09T08:17:48,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,702 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268696) roll requested 2024-12-09T08:17:48,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268702 2024-12-09T08:17:48,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268696 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268702 2024-12-09T08:17:48,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268702) roll requested 2024-12-09T08:17:48,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268709 2024-12-09T08:17:48,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268709 2024-12-09T08:17:48,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,721 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268709) roll requested 2024-12-09T08:17:48,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268721 2024-12-09T08:17:48,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268721 2024-12-09T08:17:48,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,728 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268721) roll requested 2024-12-09T08:17:48,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268728 2024-12-09T08:17:48,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268728 2024-12-09T08:17:48,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268728) roll requested 2024-12-09T08:17:48,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268735 2024-12-09T08:17:48,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268728 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268735 2024-12-09T08:17:48,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,741 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268735) roll requested 2024-12-09T08:17:48,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268742 2024-12-09T08:17:48,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268735 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268742 2024-12-09T08:17:48,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268742) roll requested 2024-12-09T08:17:48,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268752 2024-12-09T08:17:48,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268752 2024-12-09T08:17:48,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,762 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268752) roll requested 2024-12-09T08:17:48,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268762 2024-12-09T08:17:48,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268752 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268762 2024-12-09T08:17:48,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,771 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268762) roll requested 2024-12-09T08:17:48,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268772 2024-12-09T08:17:48,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268762 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268772 2024-12-09T08:17:48,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,779 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268772) roll requested 2024-12-09T08:17:48,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268779 2024-12-09T08:17:48,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268779 2024-12-09T08:17:48,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268779) roll requested 2024-12-09T08:17:48,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268785 2024-12-09T08:17:48,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268785 2024-12-09T08:17:48,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268785) roll requested 2024-12-09T08:17:48,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268791 2024-12-09T08:17:48,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268791 2024-12-09T08:17:48,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,797 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268791) roll requested 2024-12-09T08:17:48,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268798 2024-12-09T08:17:48,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268798 2024-12-09T08:17:48,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268798) roll requested 2024-12-09T08:17:48,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268805 2024-12-09T08:17:48,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268798 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268805 2024-12-09T08:17:48,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,811 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268805) roll requested 2024-12-09T08:17:48,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268811 2024-12-09T08:17:48,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268811 2024-12-09T08:17:48,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268811) roll requested 2024-12-09T08:17:48,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268819 2024-12-09T08:17:48,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268811 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268819 2024-12-09T08:17:48,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268819) roll requested 2024-12-09T08:17:48,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268829 2024-12-09T08:17:48,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268819 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268829 2024-12-09T08:17:48,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268829) roll requested 2024-12-09T08:17:48,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268841 2024-12-09T08:17:48,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268841 2024-12-09T08:17:48,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268841) roll requested 2024-12-09T08:17:48,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268852 2024-12-09T08:17:48,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268852 2024-12-09T08:17:48,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,861 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268852) roll requested 2024-12-09T08:17:48,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268861 2024-12-09T08:17:48,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268861 2024-12-09T08:17:48,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268861) roll requested 2024-12-09T08:17:48,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268868 2024-12-09T08:17:48,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268861 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268868 2024-12-09T08:17:48,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268868) roll requested 2024-12-09T08:17:48,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268877 2024-12-09T08:17:48,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268877 2024-12-09T08:17:48,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,886 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268877) roll requested 2024-12-09T08:17:48,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268887 2024-12-09T08:17:48,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268877 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268887 2024-12-09T08:17:48,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,898 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268887) roll requested 2024-12-09T08:17:48,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268898 2024-12-09T08:17:48,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268887 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268898 2024-12-09T08:17:48,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268898) roll requested 2024-12-09T08:17:48,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268908 2024-12-09T08:17:48,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268898 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268908 2024-12-09T08:17:48,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268908) roll requested 2024-12-09T08:17:48,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268921 2024-12-09T08:17:48,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268921 2024-12-09T08:17:48,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268921) roll requested 2024-12-09T08:17:48,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268928 2024-12-09T08:17:48,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268928 2024-12-09T08:17:48,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268928) roll requested 2024-12-09T08:17:48,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268936 2024-12-09T08:17:48,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268936 2024-12-09T08:17:48,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268936) roll requested 2024-12-09T08:17:48,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268947 2024-12-09T08:17:48,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268947 2024-12-09T08:17:48,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,962 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268947) roll requested 2024-12-09T08:17:48,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268962 2024-12-09T08:17:48,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268962 2024-12-09T08:17:48,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,970 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268962) roll requested 2024-12-09T08:17:48,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268970 2024-12-09T08:17:48,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268962 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268970 2024-12-09T08:17:48,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268970) roll requested 2024-12-09T08:17:48,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268982 2024-12-09T08:17:48,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268970 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268982 2024-12-09T08:17:48,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268982) roll requested 2024-12-09T08:17:48,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268989 2024-12-09T08:17:48,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:48,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268989 2024-12-09T08:17:48,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:48,995 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:48,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268989) roll requested 2024-12-09T08:17:48,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732268995 2024-12-09T08:17:49,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268995 2024-12-09T08:17:49,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732268995) roll requested 2024-12-09T08:17:49,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269004 2024-12-09T08:17:49,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732268995 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269004 2024-12-09T08:17:49,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269004) roll requested 2024-12-09T08:17:49,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269011 2024-12-09T08:17:49,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269011 2024-12-09T08:17:49,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269011) roll requested 2024-12-09T08:17:49,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269018 2024-12-09T08:17:49,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269018 2024-12-09T08:17:49,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269018) roll requested 2024-12-09T08:17:49,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269025 2024-12-09T08:17:49,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269025 2024-12-09T08:17:49,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,031 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269025) roll requested 2024-12-09T08:17:49,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269032 2024-12-09T08:17:49,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269032 2024-12-09T08:17:49,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,039 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269032) roll requested 2024-12-09T08:17:49,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269040 2024-12-09T08:17:49,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269040 2024-12-09T08:17:49,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,046 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269040) roll requested 2024-12-09T08:17:49,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269047 2024-12-09T08:17:49,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269047 2024-12-09T08:17:49,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,053 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269047) roll requested 2024-12-09T08:17:49,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269054 2024-12-09T08:17:49,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269054 2024-12-09T08:17:49,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269054) roll requested 2024-12-09T08:17:49,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269062 2024-12-09T08:17:49,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269062 2024-12-09T08:17:49,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269062) roll requested 2024-12-09T08:17:49,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269073 2024-12-09T08:17:49,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269073 2024-12-09T08:17:49,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269073) roll requested 2024-12-09T08:17:49,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269083 2024-12-09T08:17:49,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269083 2024-12-09T08:17:49,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269083) roll requested 2024-12-09T08:17:49,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269092 2024-12-09T08:17:49,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269092 2024-12-09T08:17:49,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269092) roll requested 2024-12-09T08:17:49,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269098 2024-12-09T08:17:49,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269092 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269098 2024-12-09T08:17:49,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,107 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269098) roll requested 2024-12-09T08:17:49,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269107 2024-12-09T08:17:49,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269098 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269107 2024-12-09T08:17:49,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,114 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269107) roll requested 2024-12-09T08:17:49,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269114 2024-12-09T08:17:49,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269114 2024-12-09T08:17:49,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269114) roll requested 2024-12-09T08:17:49,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269121 2024-12-09T08:17:49,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269114 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269121 2024-12-09T08:17:49,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269121) roll requested 2024-12-09T08:17:49,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269129 2024-12-09T08:17:49,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269129 2024-12-09T08:17:49,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269129) roll requested 2024-12-09T08:17:49,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269137 2024-12-09T08:17:49,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269137 2024-12-09T08:17:49,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269137) roll requested 2024-12-09T08:17:49,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269143 2024-12-09T08:17:49,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269143 2024-12-09T08:17:49,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269143) roll requested 2024-12-09T08:17:49,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269152 2024-12-09T08:17:49,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269152 2024-12-09T08:17:49,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,161 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269152) roll requested 2024-12-09T08:17:49,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269161 2024-12-09T08:17:49,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269161 2024-12-09T08:17:49,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269161) roll requested 2024-12-09T08:17:49,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269169 2024-12-09T08:17:49,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269169 2024-12-09T08:17:49,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269169) roll requested 2024-12-09T08:17:49,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269177 2024-12-09T08:17:49,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269177 2024-12-09T08:17:49,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269177) roll requested 2024-12-09T08:17:49,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269184 2024-12-09T08:17:49,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269184 2024-12-09T08:17:49,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,193 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269184) roll requested 2024-12-09T08:17:49,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269193 2024-12-09T08:17:49,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269193 2024-12-09T08:17:49,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269193) roll requested 2024-12-09T08:17:49,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269201 2024-12-09T08:17:49,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269201 2024-12-09T08:17:49,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269201) roll requested 2024-12-09T08:17:49,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269209 2024-12-09T08:17:49,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269209 2024-12-09T08:17:49,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269209) roll requested 2024-12-09T08:17:49,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269215 2024-12-09T08:17:49,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269215 2024-12-09T08:17:49,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269215) roll requested 2024-12-09T08:17:49,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269225 2024-12-09T08:17:49,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269225 2024-12-09T08:17:49,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269225) roll requested 2024-12-09T08:17:49,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269232 2024-12-09T08:17:49,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269232 2024-12-09T08:17:49,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269232) roll requested 2024-12-09T08:17:49,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269238 2024-12-09T08:17:49,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269238 2024-12-09T08:17:49,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269238) roll requested 2024-12-09T08:17:49,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269244 2024-12-09T08:17:49,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269244 2024-12-09T08:17:49,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,250 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269244) roll requested 2024-12-09T08:17:49,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269250 2024-12-09T08:17:49,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269250 2024-12-09T08:17:49,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269250) roll requested 2024-12-09T08:17:49,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269257 2024-12-09T08:17:49,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269250 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269257 2024-12-09T08:17:49,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,265 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269257) roll requested 2024-12-09T08:17:49,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269265 2024-12-09T08:17:49,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269265 2024-12-09T08:17:49,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,274 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269265) roll requested 2024-12-09T08:17:49,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269275 2024-12-09T08:17:49,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269275 2024-12-09T08:17:49,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269275) roll requested 2024-12-09T08:17:49,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269281 2024-12-09T08:17:49,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269281 2024-12-09T08:17:49,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269281) roll requested 2024-12-09T08:17:49,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269288 2024-12-09T08:17:49,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269288 2024-12-09T08:17:49,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,298 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269288) roll requested 2024-12-09T08:17:49,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269298 2024-12-09T08:17:49,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269298 2024-12-09T08:17:49,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,307 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269298) roll requested 2024-12-09T08:17:49,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269308 2024-12-09T08:17:49,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269308 2024-12-09T08:17:49,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269308) roll requested 2024-12-09T08:17:49,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269313 2024-12-09T08:17:49,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269313 2024-12-09T08:17:49,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,319 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269313) roll requested 2024-12-09T08:17:49,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269320 2024-12-09T08:17:49,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269320 2024-12-09T08:17:49,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,326 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269320) roll requested 2024-12-09T08:17:49,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269327 2024-12-09T08:17:49,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269327 2024-12-09T08:17:49,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,333 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269327) roll requested 2024-12-09T08:17:49,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269333 2024-12-09T08:17:49,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269333 2024-12-09T08:17:49,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269333) roll requested 2024-12-09T08:17:49,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269339 2024-12-09T08:17:49,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269339 2024-12-09T08:17:49,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269339) roll requested 2024-12-09T08:17:49,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269345 2024-12-09T08:17:49,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269345 2024-12-09T08:17:49,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269345) roll requested 2024-12-09T08:17:49,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269355 2024-12-09T08:17:49,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269355 2024-12-09T08:17:49,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269355) roll requested 2024-12-09T08:17:49,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269363 2024-12-09T08:17:49,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269363 2024-12-09T08:17:49,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269363) roll requested 2024-12-09T08:17:49,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269375 2024-12-09T08:17:49,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269375 2024-12-09T08:17:49,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269375) roll requested 2024-12-09T08:17:49,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269384 2024-12-09T08:17:49,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269375 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269384 2024-12-09T08:17:49,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269384) roll requested 2024-12-09T08:17:49,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269389 2024-12-09T08:17:49,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269389 2024-12-09T08:17:49,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269389) roll requested 2024-12-09T08:17:49,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269395 2024-12-09T08:17:49,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269395 2024-12-09T08:17:49,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,401 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269395) roll requested 2024-12-09T08:17:49,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269401 2024-12-09T08:17:49,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269395 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269401 2024-12-09T08:17:49,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,408 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269401) roll requested 2024-12-09T08:17:49,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269408 2024-12-09T08:17:49,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269408 2024-12-09T08:17:49,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,416 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269408) roll requested 2024-12-09T08:17:49,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269416 2024-12-09T08:17:49,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269408 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269416 2024-12-09T08:17:49,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,425 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269416) roll requested 2024-12-09T08:17:49,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269425 2024-12-09T08:17:49,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269416 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269425 2024-12-09T08:17:49,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,433 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269425) roll requested 2024-12-09T08:17:49,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269433 2024-12-09T08:17:49,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269433 2024-12-09T08:17:49,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269433) roll requested 2024-12-09T08:17:49,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269439 2024-12-09T08:17:49,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269439 2024-12-09T08:17:49,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269439) roll requested 2024-12-09T08:17:49,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269447 2024-12-09T08:17:49,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269447 2024-12-09T08:17:49,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269447) roll requested 2024-12-09T08:17:49,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269457 2024-12-09T08:17:49,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269457 2024-12-09T08:17:49,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269457) roll requested 2024-12-09T08:17:49,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269464 2024-12-09T08:17:49,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269464 2024-12-09T08:17:49,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269464) roll requested 2024-12-09T08:17:49,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269472 2024-12-09T08:17:49,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269464 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269472 2024-12-09T08:17:49,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,477 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269472) roll requested 2024-12-09T08:17:49,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269478 2024-12-09T08:17:49,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269478 2024-12-09T08:17:49,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269478) roll requested 2024-12-09T08:17:49,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269487 2024-12-09T08:17:49,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269487 2024-12-09T08:17:49,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269487) roll requested 2024-12-09T08:17:49,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269504 2024-12-09T08:17:49,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269504 2024-12-09T08:17:49,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269504) roll requested 2024-12-09T08:17:49,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269515 2024-12-09T08:17:49,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269504 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269515 2024-12-09T08:17:49,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269515) roll requested 2024-12-09T08:17:49,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269521 2024-12-09T08:17:49,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269521 2024-12-09T08:17:49,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269521) roll requested 2024-12-09T08:17:49,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269542 2024-12-09T08:17:49,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269542 2024-12-09T08:17:49,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,551 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269542) roll requested 2024-12-09T08:17:49,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269551 2024-12-09T08:17:49,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269542 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269551 2024-12-09T08:17:49,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269551) roll requested 2024-12-09T08:17:49,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269558 2024-12-09T08:17:49,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269551 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269558 2024-12-09T08:17:49,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269558) roll requested 2024-12-09T08:17:49,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269566 2024-12-09T08:17:49,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269558 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269566 2024-12-09T08:17:49,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269566) roll requested 2024-12-09T08:17:49,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269577 2024-12-09T08:17:49,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269566 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269577 2024-12-09T08:17:49,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269577) roll requested 2024-12-09T08:17:49,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269583 2024-12-09T08:17:49,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269583 2024-12-09T08:17:49,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269583) roll requested 2024-12-09T08:17:49,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269592 2024-12-09T08:17:49,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269583 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269592 2024-12-09T08:17:49,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269592) roll requested 2024-12-09T08:17:49,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269599 2024-12-09T08:17:49,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269599 2024-12-09T08:17:49,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,607 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269599) roll requested 2024-12-09T08:17:49,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269608 2024-12-09T08:17:49,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269608 2024-12-09T08:17:49,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269608) roll requested 2024-12-09T08:17:49,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269614 2024-12-09T08:17:49,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269614 2024-12-09T08:17:49,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269614) roll requested 2024-12-09T08:17:49,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269620 2024-12-09T08:17:49,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269620 2024-12-09T08:17:49,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269620) roll requested 2024-12-09T08:17:49,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269625 2024-12-09T08:17:49,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269625 2024-12-09T08:17:49,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269625) roll requested 2024-12-09T08:17:49,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269631 2024-12-09T08:17:49,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269631 2024-12-09T08:17:49,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,638 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269631) roll requested 2024-12-09T08:17:49,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269638 2024-12-09T08:17:49,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269638 2024-12-09T08:17:49,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269638) roll requested 2024-12-09T08:17:49,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269647 2024-12-09T08:17:49,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269647 2024-12-09T08:17:49,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269647) roll requested 2024-12-09T08:17:49,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269656 2024-12-09T08:17:49,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269656 2024-12-09T08:17:49,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269656) roll requested 2024-12-09T08:17:49,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269664 2024-12-09T08:17:49,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269664 2024-12-09T08:17:49,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269664) roll requested 2024-12-09T08:17:49,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269673 2024-12-09T08:17:49,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269673 2024-12-09T08:17:49,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,681 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269673) roll requested 2024-12-09T08:17:49,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269682 2024-12-09T08:17:49,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269682 2024-12-09T08:17:49,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269682) roll requested 2024-12-09T08:17:49,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269690 2024-12-09T08:17:49,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269682 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269690 2024-12-09T08:17:49,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269690) roll requested 2024-12-09T08:17:49,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269701 2024-12-09T08:17:49,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269701 2024-12-09T08:17:49,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,710 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269701) roll requested 2024-12-09T08:17:49,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269710 2024-12-09T08:17:49,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269710 2024-12-09T08:17:49,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269710) roll requested 2024-12-09T08:17:49,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269719 2024-12-09T08:17:49,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269710 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269719 2024-12-09T08:17:49,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269719) roll requested 2024-12-09T08:17:49,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269726 2024-12-09T08:17:49,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269726 2024-12-09T08:17:49,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269726) roll requested 2024-12-09T08:17:49,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269734 2024-12-09T08:17:49,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269734 2024-12-09T08:17:49,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,741 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269734) roll requested 2024-12-09T08:17:49,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269741 2024-12-09T08:17:49,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269741 2024-12-09T08:17:49,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,748 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269741) roll requested 2024-12-09T08:17:49,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269749 2024-12-09T08:17:49,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269741 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269749 2024-12-09T08:17:49,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,757 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269749) roll requested 2024-12-09T08:17:49,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269758 2024-12-09T08:17:49,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269758 2024-12-09T08:17:49,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,767 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269758) roll requested 2024-12-09T08:17:49,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269767 2024-12-09T08:17:49,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269767 2024-12-09T08:17:49,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,779 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269767) roll requested 2024-12-09T08:17:49,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269780 2024-12-09T08:17:49,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269780 2024-12-09T08:17:49,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269780) roll requested 2024-12-09T08:17:49,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269789 2024-12-09T08:17:49,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269789 2024-12-09T08:17:49,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269789) roll requested 2024-12-09T08:17:49,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269803 2024-12-09T08:17:49,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269803 2024-12-09T08:17:49,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269803) roll requested 2024-12-09T08:17:49,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269813 2024-12-09T08:17:49,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269813 2024-12-09T08:17:49,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269813) roll requested 2024-12-09T08:17:49,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269822 2024-12-09T08:17:49,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269822 2024-12-09T08:17:49,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269822) roll requested 2024-12-09T08:17:49,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269827 2024-12-09T08:17:49,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269827 2024-12-09T08:17:49,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,834 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269827) roll requested 2024-12-09T08:17:49,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269834 2024-12-09T08:17:49,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269834 2024-12-09T08:17:49,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269834) roll requested 2024-12-09T08:17:49,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269841 2024-12-09T08:17:49,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269841 2024-12-09T08:17:49,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,851 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269841) roll requested 2024-12-09T08:17:49,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269851 2024-12-09T08:17:49,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269851 2024-12-09T08:17:49,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269851) roll requested 2024-12-09T08:17:49,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269857 2024-12-09T08:17:49,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269857 2024-12-09T08:17:49,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269857) roll requested 2024-12-09T08:17:49,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269862 2024-12-09T08:17:49,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269862 2024-12-09T08:17:49,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,870 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269862) roll requested 2024-12-09T08:17:49,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269870 2024-12-09T08:17:49,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269870 2024-12-09T08:17:49,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,878 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269870) roll requested 2024-12-09T08:17:49,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269878 2024-12-09T08:17:49,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269870 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269878 2024-12-09T08:17:49,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,886 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269878) roll requested 2024-12-09T08:17:49,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269887 2024-12-09T08:17:49,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269887 2024-12-09T08:17:49,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,892 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269887) roll requested 2024-12-09T08:17:49,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269892 2024-12-09T08:17:49,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269887 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269892 2024-12-09T08:17:49,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,897 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269892) roll requested 2024-12-09T08:17:49,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269898 2024-12-09T08:17:49,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269892 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269898 2024-12-09T08:17:49,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269898) roll requested 2024-12-09T08:17:49,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269905 2024-12-09T08:17:49,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269898 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269905 2024-12-09T08:17:49,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,913 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269905) roll requested 2024-12-09T08:17:49,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269913 2024-12-09T08:17:49,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269913 2024-12-09T08:17:49,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,929 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269913) roll requested 2024-12-09T08:17:49,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269929 2024-12-09T08:17:49,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269913 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269929 2024-12-09T08:17:49,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269929) roll requested 2024-12-09T08:17:49,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269941 2024-12-09T08:17:49,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269941 2024-12-09T08:17:49,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269941) roll requested 2024-12-09T08:17:49,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269952 2024-12-09T08:17:49,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269941 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269952 2024-12-09T08:17:49,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269952) roll requested 2024-12-09T08:17:49,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269968 2024-12-09T08:17:49,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269968 2024-12-09T08:17:49,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269968) roll requested 2024-12-09T08:17:49,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269981 2024-12-09T08:17:49,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:49,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269981 2024-12-09T08:17:49,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:49,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:49,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269981) roll requested 2024-12-09T08:17:49,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732269992 2024-12-09T08:17:50,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269992 2024-12-09T08:17:50,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,002 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732269992) roll requested 2024-12-09T08:17:50,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270002 2024-12-09T08:17:50,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732269992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270002 2024-12-09T08:17:50,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270002) roll requested 2024-12-09T08:17:50,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270010 2024-12-09T08:17:50,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270002 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270010 2024-12-09T08:17:50,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270010) roll requested 2024-12-09T08:17:50,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270017 2024-12-09T08:17:50,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270017 2024-12-09T08:17:50,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270017) roll requested 2024-12-09T08:17:50,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270023 2024-12-09T08:17:50,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270023 2024-12-09T08:17:50,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,037 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270023) roll requested 2024-12-09T08:17:50,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270037 2024-12-09T08:17:50,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270037 2024-12-09T08:17:50,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270037) roll requested 2024-12-09T08:17:50,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270044 2024-12-09T08:17:50,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270044 2024-12-09T08:17:50,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,051 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270044) roll requested 2024-12-09T08:17:50,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270052 2024-12-09T08:17:50,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270052 2024-12-09T08:17:50,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270052) roll requested 2024-12-09T08:17:50,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270062 2024-12-09T08:17:50,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270062 2024-12-09T08:17:50,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270062) roll requested 2024-12-09T08:17:50,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270068 2024-12-09T08:17:50,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270068 2024-12-09T08:17:50,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,074 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270068) roll requested 2024-12-09T08:17:50,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270074 2024-12-09T08:17:50,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270074 2024-12-09T08:17:50,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270074) roll requested 2024-12-09T08:17:50,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270085 2024-12-09T08:17:50,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270074 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270085 2024-12-09T08:17:50,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,097 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270085) roll requested 2024-12-09T08:17:50,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270097 2024-12-09T08:17:50,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270097 2024-12-09T08:17:50,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270097) roll requested 2024-12-09T08:17:50,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270104 2024-12-09T08:17:50,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270104 2024-12-09T08:17:50,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270104) roll requested 2024-12-09T08:17:50,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270113 2024-12-09T08:17:50,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270104 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270113 2024-12-09T08:17:50,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,121 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270113) roll requested 2024-12-09T08:17:50,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270121 2024-12-09T08:17:50,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270121 2024-12-09T08:17:50,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270121) roll requested 2024-12-09T08:17:50,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270128 2024-12-09T08:17:50,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270128 2024-12-09T08:17:50,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,135 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270128) roll requested 2024-12-09T08:17:50,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270135 2024-12-09T08:17:50,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270135 2024-12-09T08:17:50,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270135) roll requested 2024-12-09T08:17:50,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270141 2024-12-09T08:17:50,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270141 2024-12-09T08:17:50,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270141) roll requested 2024-12-09T08:17:50,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270151 2024-12-09T08:17:50,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270151 2024-12-09T08:17:50,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270151) roll requested 2024-12-09T08:17:50,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270157 2024-12-09T08:17:50,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270157 2024-12-09T08:17:50,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,163 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270157) roll requested 2024-12-09T08:17:50,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270164 2024-12-09T08:17:50,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270164 2024-12-09T08:17:50,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270164) roll requested 2024-12-09T08:17:50,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270170 2024-12-09T08:17:50,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270164 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270170 2024-12-09T08:17:50,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270170) roll requested 2024-12-09T08:17:50,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270176 2024-12-09T08:17:50,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270170 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270176 2024-12-09T08:17:50,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270176) roll requested 2024-12-09T08:17:50,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270183 2024-12-09T08:17:50,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270183 2024-12-09T08:17:50,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,191 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270183) roll requested 2024-12-09T08:17:50,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270191 2024-12-09T08:17:50,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270191 2024-12-09T08:17:50,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270191) roll requested 2024-12-09T08:17:50,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270197 2024-12-09T08:17:50,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270191 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270197 2024-12-09T08:17:50,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270197) roll requested 2024-12-09T08:17:50,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270204 2024-12-09T08:17:50,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270204 2024-12-09T08:17:50,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270204) roll requested 2024-12-09T08:17:50,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270210 2024-12-09T08:17:50,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270210 2024-12-09T08:17:50,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270210) roll requested 2024-12-09T08:17:50,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270216 2024-12-09T08:17:50,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270210 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270216 2024-12-09T08:17:50,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270216) roll requested 2024-12-09T08:17:50,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270222 2024-12-09T08:17:50,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270222 2024-12-09T08:17:50,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270222) roll requested 2024-12-09T08:17:50,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270228 2024-12-09T08:17:50,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270228 2024-12-09T08:17:50,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270228) roll requested 2024-12-09T08:17:50,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270234 2024-12-09T08:17:50,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270234 2024-12-09T08:17:50,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270234) roll requested 2024-12-09T08:17:50,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270241 2024-12-09T08:17:50,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270241 2024-12-09T08:17:50,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270241) roll requested 2024-12-09T08:17:50,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270247 2024-12-09T08:17:50,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270241 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270247 2024-12-09T08:17:50,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270247) roll requested 2024-12-09T08:17:50,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270252 2024-12-09T08:17:50,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270247 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270252 2024-12-09T08:17:50,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,258 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270252) roll requested 2024-12-09T08:17:50,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270259 2024-12-09T08:17:50,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270259 2024-12-09T08:17:50,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,265 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270259) roll requested 2024-12-09T08:17:50,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270265 2024-12-09T08:17:50,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270265 2024-12-09T08:17:50,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,271 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270265) roll requested 2024-12-09T08:17:50,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270271 2024-12-09T08:17:50,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270271 2024-12-09T08:17:50,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,282 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270271) roll requested 2024-12-09T08:17:50,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270282 2024-12-09T08:17:50,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270282 2024-12-09T08:17:50,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270282) roll requested 2024-12-09T08:17:50,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270293 2024-12-09T08:17:50,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270282 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270293 2024-12-09T08:17:50,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270293) roll requested 2024-12-09T08:17:50,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270299 2024-12-09T08:17:50,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270299 2024-12-09T08:17:50,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270299) roll requested 2024-12-09T08:17:50,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270305 2024-12-09T08:17:50,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270305 2024-12-09T08:17:50,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270305) roll requested 2024-12-09T08:17:50,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270314 2024-12-09T08:17:50,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270314 2024-12-09T08:17:50,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,319 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270314) roll requested 2024-12-09T08:17:50,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270320 2024-12-09T08:17:50,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270320 2024-12-09T08:17:50,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,325 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270320) roll requested 2024-12-09T08:17:50,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270325 2024-12-09T08:17:50,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270325 2024-12-09T08:17:50,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,330 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270325) roll requested 2024-12-09T08:17:50,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270331 2024-12-09T08:17:50,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270331 2024-12-09T08:17:50,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270331) roll requested 2024-12-09T08:17:50,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270341 2024-12-09T08:17:50,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270341 2024-12-09T08:17:50,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,349 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270341) roll requested 2024-12-09T08:17:50,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270350 2024-12-09T08:17:50,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270350 2024-12-09T08:17:50,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,355 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270350) roll requested 2024-12-09T08:17:50,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270355 2024-12-09T08:17:50,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270355 2024-12-09T08:17:50,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270355) roll requested 2024-12-09T08:17:50,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270361 2024-12-09T08:17:50,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270361 2024-12-09T08:17:50,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,366 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270361) roll requested 2024-12-09T08:17:50,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270367 2024-12-09T08:17:50,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270367 2024-12-09T08:17:50,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270367) roll requested 2024-12-09T08:17:50,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270372 2024-12-09T08:17:50,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270367 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270372 2024-12-09T08:17:50,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270372) roll requested 2024-12-09T08:17:50,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270378 2024-12-09T08:17:50,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270378 2024-12-09T08:17:50,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270378) roll requested 2024-12-09T08:17:50,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270384 2024-12-09T08:17:50,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270384 2024-12-09T08:17:50,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270384) roll requested 2024-12-09T08:17:50,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270397 2024-12-09T08:17:50,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270397 2024-12-09T08:17:50,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270397) roll requested 2024-12-09T08:17:50,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270404 2024-12-09T08:17:50,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270404 2024-12-09T08:17:50,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,410 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270404) roll requested 2024-12-09T08:17:50,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270410 2024-12-09T08:17:50,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270410 2024-12-09T08:17:50,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270410) roll requested 2024-12-09T08:17:50,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270416 2024-12-09T08:17:50,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270410 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270416 2024-12-09T08:17:50,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270416) roll requested 2024-12-09T08:17:50,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270423 2024-12-09T08:17:50,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270416 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270423 2024-12-09T08:17:50,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,430 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270423) roll requested 2024-12-09T08:17:50,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270431 2024-12-09T08:17:50,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270423 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270431 2024-12-09T08:17:50,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270431) roll requested 2024-12-09T08:17:50,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270440 2024-12-09T08:17:50,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270431 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270440 2024-12-09T08:17:50,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270440) roll requested 2024-12-09T08:17:50,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270450 2024-12-09T08:17:50,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270450 2024-12-09T08:17:50,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270450) roll requested 2024-12-09T08:17:50,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270461 2024-12-09T08:17:50,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270461 2024-12-09T08:17:50,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,469 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270461) roll requested 2024-12-09T08:17:50,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270469 2024-12-09T08:17:50,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270469 2024-12-09T08:17:50,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,477 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270469) roll requested 2024-12-09T08:17:50,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270477 2024-12-09T08:17:50,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270469 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270477 2024-12-09T08:17:50,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,485 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270477) roll requested 2024-12-09T08:17:50,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270485 2024-12-09T08:17:50,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270477 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270485 2024-12-09T08:17:50,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,498 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270485) roll requested 2024-12-09T08:17:50,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270498 2024-12-09T08:17:50,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270485 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270498 2024-12-09T08:17:50,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270498) roll requested 2024-12-09T08:17:50,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270504 2024-12-09T08:17:50,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270498 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270504 2024-12-09T08:17:50,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,510 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270504) roll requested 2024-12-09T08:17:50,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270510 2024-12-09T08:17:50,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270504 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270510 2024-12-09T08:17:50,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270510) roll requested 2024-12-09T08:17:50,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270517 2024-12-09T08:17:50,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270510 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270517 2024-12-09T08:17:50,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,525 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270517) roll requested 2024-12-09T08:17:50,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270525 2024-12-09T08:17:50,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270525 2024-12-09T08:17:50,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,534 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270525) roll requested 2024-12-09T08:17:50,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270534 2024-12-09T08:17:50,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270525 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270534 2024-12-09T08:17:50,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270534) roll requested 2024-12-09T08:17:50,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270540 2024-12-09T08:17:50,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270534 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270540 2024-12-09T08:17:50,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,545 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270540) roll requested 2024-12-09T08:17:50,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270546 2024-12-09T08:17:50,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270540 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270546 2024-12-09T08:17:50,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270546) roll requested 2024-12-09T08:17:50,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270552 2024-12-09T08:17:50,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270546 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270552 2024-12-09T08:17:50,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,558 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270552) roll requested 2024-12-09T08:17:50,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270558 2024-12-09T08:17:50,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270558 2024-12-09T08:17:50,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270558) roll requested 2024-12-09T08:17:50,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270566 2024-12-09T08:17:50,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270558 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270566 2024-12-09T08:17:50,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270566) roll requested 2024-12-09T08:17:50,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270571 2024-12-09T08:17:50,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270566 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270571 2024-12-09T08:17:50,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270571) roll requested 2024-12-09T08:17:50,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270577 2024-12-09T08:17:50,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270577 2024-12-09T08:17:50,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,582 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270577) roll requested 2024-12-09T08:17:50,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270583 2024-12-09T08:17:50,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270583 2024-12-09T08:17:50,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270583) roll requested 2024-12-09T08:17:50,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270589 2024-12-09T08:17:50,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270583 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270589 2024-12-09T08:17:50,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270589) roll requested 2024-12-09T08:17:50,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270604 2024-12-09T08:17:50,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270604 2024-12-09T08:17:50,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270604) roll requested 2024-12-09T08:17:50,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270609 2024-12-09T08:17:50,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270609 2024-12-09T08:17:50,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,615 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270609) roll requested 2024-12-09T08:17:50,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270615 2024-12-09T08:17:50,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270615 2024-12-09T08:17:50,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270615) roll requested 2024-12-09T08:17:50,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270621 2024-12-09T08:17:50,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270615 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270621 2024-12-09T08:17:50,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270621) roll requested 2024-12-09T08:17:50,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270628 2024-12-09T08:17:50,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270628 2024-12-09T08:17:50,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,635 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270628) roll requested 2024-12-09T08:17:50,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270636 2024-12-09T08:17:50,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270636 2024-12-09T08:17:50,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,643 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270636) roll requested 2024-12-09T08:17:50,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270644 2024-12-09T08:17:50,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270636 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270644 2024-12-09T08:17:50,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270644) roll requested 2024-12-09T08:17:50,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270653 2024-12-09T08:17:50,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270653 2024-12-09T08:17:50,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270653) roll requested 2024-12-09T08:17:50,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270660 2024-12-09T08:17:50,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270660 2024-12-09T08:17:50,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270660) roll requested 2024-12-09T08:17:50,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270669 2024-12-09T08:17:50,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270669 2024-12-09T08:17:50,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,678 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270669) roll requested 2024-12-09T08:17:50,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270678 2024-12-09T08:17:50,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270678 2024-12-09T08:17:50,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270678) roll requested 2024-12-09T08:17:50,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270686 2024-12-09T08:17:50,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270686 2024-12-09T08:17:50,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,693 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270686) roll requested 2024-12-09T08:17:50,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270694 2024-12-09T08:17:50,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270686 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270694 2024-12-09T08:17:50,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270694) roll requested 2024-12-09T08:17:50,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270702 2024-12-09T08:17:50,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270694 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270702 2024-12-09T08:17:50,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270702) roll requested 2024-12-09T08:17:50,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270712 2024-12-09T08:17:50,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270712 2024-12-09T08:17:50,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270712) roll requested 2024-12-09T08:17:50,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270720 2024-12-09T08:17:50,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270720 2024-12-09T08:17:50,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270720) roll requested 2024-12-09T08:17:50,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270725 2024-12-09T08:17:50,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270725 2024-12-09T08:17:50,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270725) roll requested 2024-12-09T08:17:50,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270730 2024-12-09T08:17:50,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270730 2024-12-09T08:17:50,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270730) roll requested 2024-12-09T08:17:50,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270736 2024-12-09T08:17:50,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270730 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270736 2024-12-09T08:17:50,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,743 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270736) roll requested 2024-12-09T08:17:50,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270744 2024-12-09T08:17:50,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270744 2024-12-09T08:17:50,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270744) roll requested 2024-12-09T08:17:50,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270752 2024-12-09T08:17:50,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270752 2024-12-09T08:17:50,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,766 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270752) roll requested 2024-12-09T08:17:50,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270766 2024-12-09T08:17:50,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270752 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270766 2024-12-09T08:17:50,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270766) roll requested 2024-12-09T08:17:50,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270772 2024-12-09T08:17:50,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270766 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270772 2024-12-09T08:17:50,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270772) roll requested 2024-12-09T08:17:50,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270779 2024-12-09T08:17:50,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270779 2024-12-09T08:17:50,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270779) roll requested 2024-12-09T08:17:50,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270787 2024-12-09T08:17:50,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270787 2024-12-09T08:17:50,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270787) roll requested 2024-12-09T08:17:50,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270795 2024-12-09T08:17:50,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270787 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270795 2024-12-09T08:17:50,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,801 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270795) roll requested 2024-12-09T08:17:50,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270802 2024-12-09T08:17:50,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270802 2024-12-09T08:17:50,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270802) roll requested 2024-12-09T08:17:50,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270812 2024-12-09T08:17:50,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270802 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270812 2024-12-09T08:17:50,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270812) roll requested 2024-12-09T08:17:50,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270821 2024-12-09T08:17:50,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270821 2024-12-09T08:17:50,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,826 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270821) roll requested 2024-12-09T08:17:50,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270826 2024-12-09T08:17:50,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270826 2024-12-09T08:17:50,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,835 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270826) roll requested 2024-12-09T08:17:50,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270835 2024-12-09T08:17:50,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270835 2024-12-09T08:17:50,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,844 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270835) roll requested 2024-12-09T08:17:50,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270844 2024-12-09T08:17:50,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270835 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270844 2024-12-09T08:17:50,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270844) roll requested 2024-12-09T08:17:50,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270849 2024-12-09T08:17:50,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270849 2024-12-09T08:17:50,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270849) roll requested 2024-12-09T08:17:50,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270855 2024-12-09T08:17:50,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270855 2024-12-09T08:17:50,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,863 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270855) roll requested 2024-12-09T08:17:50,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270863 2024-12-09T08:17:50,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270855 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270863 2024-12-09T08:17:50,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,871 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270863) roll requested 2024-12-09T08:17:50,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270871 2024-12-09T08:17:50,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270863 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270871 2024-12-09T08:17:50,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,879 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270871) roll requested 2024-12-09T08:17:50,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270879 2024-12-09T08:17:50,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270879 2024-12-09T08:17:50,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,889 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270879) roll requested 2024-12-09T08:17:50,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270889 2024-12-09T08:17:50,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270879 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270889 2024-12-09T08:17:50,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270889) roll requested 2024-12-09T08:17:50,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270900 2024-12-09T08:17:50,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270900 2024-12-09T08:17:50,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270900) roll requested 2024-12-09T08:17:50,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270912 2024-12-09T08:17:50,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270912 2024-12-09T08:17:50,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270912) roll requested 2024-12-09T08:17:50,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270920 2024-12-09T08:17:50,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270920 2024-12-09T08:17:50,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270920) roll requested 2024-12-09T08:17:50,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270926 2024-12-09T08:17:50,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270926 2024-12-09T08:17:50,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270926) roll requested 2024-12-09T08:17:50,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270931 2024-12-09T08:17:50,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270926 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270931 2024-12-09T08:17:50,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270931) roll requested 2024-12-09T08:17:50,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270936 2024-12-09T08:17:50,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270936 2024-12-09T08:17:50,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270936) roll requested 2024-12-09T08:17:50,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270941 2024-12-09T08:17:50,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270941 2024-12-09T08:17:50,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270941) roll requested 2024-12-09T08:17:50,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270948 2024-12-09T08:17:50,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270941 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270948 2024-12-09T08:17:50,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,955 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270948) roll requested 2024-12-09T08:17:50,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270956 2024-12-09T08:17:50,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270948 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270956 2024-12-09T08:17:50,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270956) roll requested 2024-12-09T08:17:50,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270964 2024-12-09T08:17:50,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270956 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270964 2024-12-09T08:17:50,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270964) roll requested 2024-12-09T08:17:50,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270972 2024-12-09T08:17:50,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270972 2024-12-09T08:17:50,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270972) roll requested 2024-12-09T08:17:50,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270980 2024-12-09T08:17:50,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270980 2024-12-09T08:17:50,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270980) roll requested 2024-12-09T08:17:50,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270988 2024-12-09T08:17:50,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:50,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270988 2024-12-09T08:17:50,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:50,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:50,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270988) roll requested 2024-12-09T08:17:50,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732270999 2024-12-09T08:17:51,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270999 2024-12-09T08:17:51,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732270999) roll requested 2024-12-09T08:17:51,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271006 2024-12-09T08:17:51,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732270999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271006 2024-12-09T08:17:51,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,013 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271006) roll requested 2024-12-09T08:17:51,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271014 2024-12-09T08:17:51,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271014 2024-12-09T08:17:51,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,022 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271014) roll requested 2024-12-09T08:17:51,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271022 2024-12-09T08:17:51,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271022 2024-12-09T08:17:51,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,031 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271022) roll requested 2024-12-09T08:17:51,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271032 2024-12-09T08:17:51,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271022 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271032 2024-12-09T08:17:51,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,037 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271032) roll requested 2024-12-09T08:17:51,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271038 2024-12-09T08:17:51,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271038 2024-12-09T08:17:51,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,045 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271038) roll requested 2024-12-09T08:17:51,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271046 2024-12-09T08:17:51,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271038 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271046 2024-12-09T08:17:51,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271046) roll requested 2024-12-09T08:17:51,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271052 2024-12-09T08:17:51,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271046 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271052 2024-12-09T08:17:51,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,058 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271052) roll requested 2024-12-09T08:17:51,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271059 2024-12-09T08:17:51,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271059 2024-12-09T08:17:51,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,067 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271059) roll requested 2024-12-09T08:17:51,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271068 2024-12-09T08:17:51,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271059 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271068 2024-12-09T08:17:51,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271068) roll requested 2024-12-09T08:17:51,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271077 2024-12-09T08:17:51,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271077 2024-12-09T08:17:51,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271077) roll requested 2024-12-09T08:17:51,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271083 2024-12-09T08:17:51,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271083 2024-12-09T08:17:51,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271083) roll requested 2024-12-09T08:17:51,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271088 2024-12-09T08:17:51,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271088 2024-12-09T08:17:51,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,094 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271088) roll requested 2024-12-09T08:17:51,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271094 2024-12-09T08:17:51,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271088 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271094 2024-12-09T08:17:51,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271094) roll requested 2024-12-09T08:17:51,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271101 2024-12-09T08:17:51,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271094 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271101 2024-12-09T08:17:51,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271101) roll requested 2024-12-09T08:17:51,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271108 2024-12-09T08:17:51,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271101 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271108 2024-12-09T08:17:51,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,115 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271108) roll requested 2024-12-09T08:17:51,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271115 2024-12-09T08:17:51,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271115 2024-12-09T08:17:51,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271115) roll requested 2024-12-09T08:17:51,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271123 2024-12-09T08:17:51,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271115 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271123 2024-12-09T08:17:51,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271123) roll requested 2024-12-09T08:17:51,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271133 2024-12-09T08:17:51,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271123 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271133 2024-12-09T08:17:51,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271133) roll requested 2024-12-09T08:17:51,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271141 2024-12-09T08:17:51,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271141 2024-12-09T08:17:51,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,147 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271141) roll requested 2024-12-09T08:17:51,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271148 2024-12-09T08:17:51,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271148 2024-12-09T08:17:51,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271148) roll requested 2024-12-09T08:17:51,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271153 2024-12-09T08:17:51,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271153 2024-12-09T08:17:51,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271153) roll requested 2024-12-09T08:17:51,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271158 2024-12-09T08:17:51,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271158 2024-12-09T08:17:51,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,163 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271158) roll requested 2024-12-09T08:17:51,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271163 2024-12-09T08:17:51,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271163 2024-12-09T08:17:51,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,173 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271163) roll requested 2024-12-09T08:17:51,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271173 2024-12-09T08:17:51,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271163 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271173 2024-12-09T08:17:51,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,181 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271173) roll requested 2024-12-09T08:17:51,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271181 2024-12-09T08:17:51,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271181 2024-12-09T08:17:51,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271181) roll requested 2024-12-09T08:17:51,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271189 2024-12-09T08:17:51,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271189 2024-12-09T08:17:51,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,194 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271189) roll requested 2024-12-09T08:17:51,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271195 2024-12-09T08:17:51,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271195 2024-12-09T08:17:51,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,200 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271195) roll requested 2024-12-09T08:17:51,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271200 2024-12-09T08:17:51,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271200 2024-12-09T08:17:51,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271200) roll requested 2024-12-09T08:17:51,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271212 2024-12-09T08:17:51,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271200 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271212 2024-12-09T08:17:51,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271212) roll requested 2024-12-09T08:17:51,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271218 2024-12-09T08:17:51,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271218 2024-12-09T08:17:51,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,223 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271218) roll requested 2024-12-09T08:17:51,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271223 2024-12-09T08:17:51,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271218 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271223 2024-12-09T08:17:51,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271223) roll requested 2024-12-09T08:17:51,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271229 2024-12-09T08:17:51,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271223 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271229 2024-12-09T08:17:51,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,235 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271229) roll requested 2024-12-09T08:17:51,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271235 2024-12-09T08:17:51,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271235 2024-12-09T08:17:51,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271235) roll requested 2024-12-09T08:17:51,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271240 2024-12-09T08:17:51,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271235 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271240 2024-12-09T08:17:51,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271240) roll requested 2024-12-09T08:17:51,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271246 2024-12-09T08:17:51,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271246 2024-12-09T08:17:51,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271246) roll requested 2024-12-09T08:17:51,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271252 2024-12-09T08:17:51,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271252 2024-12-09T08:17:51,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271252) roll requested 2024-12-09T08:17:51,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271260 2024-12-09T08:17:51,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271260 2024-12-09T08:17:51,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,269 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271260) roll requested 2024-12-09T08:17:51,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271269 2024-12-09T08:17:51,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271260 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271269 2024-12-09T08:17:51,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271269) roll requested 2024-12-09T08:17:51,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271275 2024-12-09T08:17:51,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271269 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271275 2024-12-09T08:17:51,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271275) roll requested 2024-12-09T08:17:51,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271281 2024-12-09T08:17:51,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271281 2024-12-09T08:17:51,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271281) roll requested 2024-12-09T08:17:51,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271287 2024-12-09T08:17:51,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271287 2024-12-09T08:17:51,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271287) roll requested 2024-12-09T08:17:51,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271292 2024-12-09T08:17:51,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271292 2024-12-09T08:17:51,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,298 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271292) roll requested 2024-12-09T08:17:51,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271299 2024-12-09T08:17:51,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271299 2024-12-09T08:17:51,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271299) roll requested 2024-12-09T08:17:51,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271304 2024-12-09T08:17:51,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271304 2024-12-09T08:17:51,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,310 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271304) roll requested 2024-12-09T08:17:51,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271311 2024-12-09T08:17:51,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271311 2024-12-09T08:17:51,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271311) roll requested 2024-12-09T08:17:51,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271317 2024-12-09T08:17:51,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271311 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271317 2024-12-09T08:17:51,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,323 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271317) roll requested 2024-12-09T08:17:51,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271323 2024-12-09T08:17:51,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271323 2024-12-09T08:17:51,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,329 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271323) roll requested 2024-12-09T08:17:51,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271329 2024-12-09T08:17:51,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271329 2024-12-09T08:17:51,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,335 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271329) roll requested 2024-12-09T08:17:51,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271335 2024-12-09T08:17:51,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271329 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271335 2024-12-09T08:17:51,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271335) roll requested 2024-12-09T08:17:51,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271340 2024-12-09T08:17:51,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271340 2024-12-09T08:17:51,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,346 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271340) roll requested 2024-12-09T08:17:51,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271346 2024-12-09T08:17:51,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271346 2024-12-09T08:17:51,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,351 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271346) roll requested 2024-12-09T08:17:51,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271351 2024-12-09T08:17:51,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271346 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271351 2024-12-09T08:17:51,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271351) roll requested 2024-12-09T08:17:51,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271356 2024-12-09T08:17:51,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271351 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271356 2024-12-09T08:17:51,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271356) roll requested 2024-12-09T08:17:51,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271362 2024-12-09T08:17:51,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271362 2024-12-09T08:17:51,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,367 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271362) roll requested 2024-12-09T08:17:51,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271368 2024-12-09T08:17:51,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271368 2024-12-09T08:17:51,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271368) roll requested 2024-12-09T08:17:51,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271375 2024-12-09T08:17:51,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271375 2024-12-09T08:17:51,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271375) roll requested 2024-12-09T08:17:51,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271383 2024-12-09T08:17:51,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271375 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271383 2024-12-09T08:17:51,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,391 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271383) roll requested 2024-12-09T08:17:51,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271391 2024-12-09T08:17:51,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271391 2024-12-09T08:17:51,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271391) roll requested 2024-12-09T08:17:51,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271400 2024-12-09T08:17:51,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271391 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271400 2024-12-09T08:17:51,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,410 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271400) roll requested 2024-12-09T08:17:51,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271411 2024-12-09T08:17:51,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271411 2024-12-09T08:17:51,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,416 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271411) roll requested 2024-12-09T08:17:51,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271416 2024-12-09T08:17:51,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271416 2024-12-09T08:17:51,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271416) roll requested 2024-12-09T08:17:51,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271422 2024-12-09T08:17:51,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271416 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271422 2024-12-09T08:17:51,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271422) roll requested 2024-12-09T08:17:51,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271428 2024-12-09T08:17:51,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271428 2024-12-09T08:17:51,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,435 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271428) roll requested 2024-12-09T08:17:51,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271435 2024-12-09T08:17:51,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271435 2024-12-09T08:17:51,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271435) roll requested 2024-12-09T08:17:51,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271441 2024-12-09T08:17:51,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271435 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271441 2024-12-09T08:17:51,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271441) roll requested 2024-12-09T08:17:51,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271446 2024-12-09T08:17:51,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271446 2024-12-09T08:17:51,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,451 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271446) roll requested 2024-12-09T08:17:51,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271451 2024-12-09T08:17:51,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271446 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271451 2024-12-09T08:17:51,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271451) roll requested 2024-12-09T08:17:51,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271457 2024-12-09T08:17:51,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271451 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271457 2024-12-09T08:17:51,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271457) roll requested 2024-12-09T08:17:51,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271465 2024-12-09T08:17:51,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271465 2024-12-09T08:17:51,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271465) roll requested 2024-12-09T08:17:51,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271475 2024-12-09T08:17:51,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271475 2024-12-09T08:17:51,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271475) roll requested 2024-12-09T08:17:51,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271483 2024-12-09T08:17:51,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271475 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271483 2024-12-09T08:17:51,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271483) roll requested 2024-12-09T08:17:51,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271488 2024-12-09T08:17:51,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271488 2024-12-09T08:17:51,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,493 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271488) roll requested 2024-12-09T08:17:51,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271493 2024-12-09T08:17:51,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271493 2024-12-09T08:17:51,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271493) roll requested 2024-12-09T08:17:51,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271502 2024-12-09T08:17:51,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271502 2024-12-09T08:17:51,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,511 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271502) roll requested 2024-12-09T08:17:51,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271512 2024-12-09T08:17:51,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271512 2024-12-09T08:17:51,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,517 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271512) roll requested 2024-12-09T08:17:51,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271517 2024-12-09T08:17:51,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271517 2024-12-09T08:17:51,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271517) roll requested 2024-12-09T08:17:51,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271522 2024-12-09T08:17:51,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271522 2024-12-09T08:17:51,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271522) roll requested 2024-12-09T08:17:51,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271527 2024-12-09T08:17:51,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271527 2024-12-09T08:17:51,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,532 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271527) roll requested 2024-12-09T08:17:51,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271532 2024-12-09T08:17:51,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271532 2024-12-09T08:17:51,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271532) roll requested 2024-12-09T08:17:51,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271539 2024-12-09T08:17:51,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271532 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271539 2024-12-09T08:17:51,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,545 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271539) roll requested 2024-12-09T08:17:51,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271545 2024-12-09T08:17:51,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271545 2024-12-09T08:17:51,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271545) roll requested 2024-12-09T08:17:51,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271553 2024-12-09T08:17:51,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271545 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271553 2024-12-09T08:17:51,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271553) roll requested 2024-12-09T08:17:51,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271559 2024-12-09T08:17:51,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271559 2024-12-09T08:17:51,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271559) roll requested 2024-12-09T08:17:51,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271566 2024-12-09T08:17:51,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271566 2024-12-09T08:17:51,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,574 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271566) roll requested 2024-12-09T08:17:51,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271574 2024-12-09T08:17:51,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271566 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271574 2024-12-09T08:17:51,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,582 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271574) roll requested 2024-12-09T08:17:51,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271583 2024-12-09T08:17:51,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271574 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271583 2024-12-09T08:17:51,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271583) roll requested 2024-12-09T08:17:51,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271592 2024-12-09T08:17:51,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271583 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271592 2024-12-09T08:17:51,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271592) roll requested 2024-12-09T08:17:51,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271601 2024-12-09T08:17:51,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271601 2024-12-09T08:17:51,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271601) roll requested 2024-12-09T08:17:51,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271608 2024-12-09T08:17:51,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271608 2024-12-09T08:17:51,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,615 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271608) roll requested 2024-12-09T08:17:51,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271615 2024-12-09T08:17:51,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271615 2024-12-09T08:17:51,622 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,622 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271615) roll requested 2024-12-09T08:17:51,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271623 2024-12-09T08:17:51,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271615 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271623 2024-12-09T08:17:51,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,629 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271623) roll requested 2024-12-09T08:17:51,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271630 2024-12-09T08:17:51,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271623 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271630 2024-12-09T08:17:51,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271630) roll requested 2024-12-09T08:17:51,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271637 2024-12-09T08:17:51,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271630 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271637 2024-12-09T08:17:51,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271637) roll requested 2024-12-09T08:17:51,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271648 2024-12-09T08:17:51,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271637 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271648 2024-12-09T08:17:51,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271648) roll requested 2024-12-09T08:17:51,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271654 2024-12-09T08:17:51,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271654 2024-12-09T08:17:51,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271654) roll requested 2024-12-09T08:17:51,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271662 2024-12-09T08:17:51,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271662 2024-12-09T08:17:51,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271662) roll requested 2024-12-09T08:17:51,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271668 2024-12-09T08:17:51,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271668 2024-12-09T08:17:51,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,673 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271668) roll requested 2024-12-09T08:17:51,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271673 2024-12-09T08:17:51,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271673 2024-12-09T08:17:51,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,678 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271673) roll requested 2024-12-09T08:17:51,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271679 2024-12-09T08:17:51,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271679 2024-12-09T08:17:51,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271679) roll requested 2024-12-09T08:17:51,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271684 2024-12-09T08:17:51,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271679 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271684 2024-12-09T08:17:51,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,691 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271684) roll requested 2024-12-09T08:17:51,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271691 2024-12-09T08:17:51,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271691 2024-12-09T08:17:51,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,699 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271691) roll requested 2024-12-09T08:17:51,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271699 2024-12-09T08:17:51,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271691 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271699 2024-12-09T08:17:51,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271699) roll requested 2024-12-09T08:17:51,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271709 2024-12-09T08:17:51,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271699 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271709 2024-12-09T08:17:51,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271709) roll requested 2024-12-09T08:17:51,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271719 2024-12-09T08:17:51,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271719 2024-12-09T08:17:51,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,728 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271719) roll requested 2024-12-09T08:17:51,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271728 2024-12-09T08:17:51,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271728 2024-12-09T08:17:51,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271728) roll requested 2024-12-09T08:17:51,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271735 2024-12-09T08:17:51,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271728 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271735 2024-12-09T08:17:51,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271735) roll requested 2024-12-09T08:17:51,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271744 2024-12-09T08:17:51,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271735 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271744 2024-12-09T08:17:51,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271744) roll requested 2024-12-09T08:17:51,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271750 2024-12-09T08:17:51,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271750 2024-12-09T08:17:51,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271750) roll requested 2024-12-09T08:17:51,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271759 2024-12-09T08:17:51,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271759 2024-12-09T08:17:51,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271759) roll requested 2024-12-09T08:17:51,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271765 2024-12-09T08:17:51,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271765 2024-12-09T08:17:51,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271765) roll requested 2024-12-09T08:17:51,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271771 2024-12-09T08:17:51,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271771 2024-12-09T08:17:51,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271771) roll requested 2024-12-09T08:17:51,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271776 2024-12-09T08:17:51,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271771 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271776 2024-12-09T08:17:51,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,782 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271776) roll requested 2024-12-09T08:17:51,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271782 2024-12-09T08:17:51,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271782 2024-12-09T08:17:51,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,787 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271782) roll requested 2024-12-09T08:17:51,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271788 2024-12-09T08:17:51,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271782 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271788 2024-12-09T08:17:51,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,794 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271788) roll requested 2024-12-09T08:17:51,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271795 2024-12-09T08:17:51,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271795 2024-12-09T08:17:51,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,801 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271795) roll requested 2024-12-09T08:17:51,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271802 2024-12-09T08:17:51,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271802 2024-12-09T08:17:51,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271802) roll requested 2024-12-09T08:17:51,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271809 2024-12-09T08:17:51,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271802 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271809 2024-12-09T08:17:51,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,814 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271809) roll requested 2024-12-09T08:17:51,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271815 2024-12-09T08:17:51,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271809 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271815 2024-12-09T08:17:51,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271815) roll requested 2024-12-09T08:17:51,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271822 2024-12-09T08:17:51,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271815 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271822 2024-12-09T08:17:51,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,830 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271822) roll requested 2024-12-09T08:17:51,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271830 2024-12-09T08:17:51,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271830 2024-12-09T08:17:51,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271830) roll requested 2024-12-09T08:17:51,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271840 2024-12-09T08:17:51,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271830 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271840 2024-12-09T08:17:51,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,850 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271840) roll requested 2024-12-09T08:17:51,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271850 2024-12-09T08:17:51,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271850 2024-12-09T08:17:51,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,861 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271850) roll requested 2024-12-09T08:17:51,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271861 2024-12-09T08:17:51,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271861 2024-12-09T08:17:51,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,866 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271861) roll requested 2024-12-09T08:17:51,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271866 2024-12-09T08:17:51,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271861 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271866 2024-12-09T08:17:51,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,871 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271866) roll requested 2024-12-09T08:17:51,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271871 2024-12-09T08:17:51,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271866 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271871 2024-12-09T08:17:51,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271871) roll requested 2024-12-09T08:17:51,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271876 2024-12-09T08:17:51,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271876 2024-12-09T08:17:51,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271876) roll requested 2024-12-09T08:17:51,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271882 2024-12-09T08:17:51,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271882 2024-12-09T08:17:51,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,889 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271882) roll requested 2024-12-09T08:17:51,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271889 2024-12-09T08:17:51,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271882 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271889 2024-12-09T08:17:51,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,898 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271889) roll requested 2024-12-09T08:17:51,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271898 2024-12-09T08:17:51,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271898 2024-12-09T08:17:51,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,905 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271898) roll requested 2024-12-09T08:17:51,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271905 2024-12-09T08:17:51,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271898 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271905 2024-12-09T08:17:51,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271905) roll requested 2024-12-09T08:17:51,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271915 2024-12-09T08:17:51,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271915 2024-12-09T08:17:51,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,923 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271915) roll requested 2024-12-09T08:17:51,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271923 2024-12-09T08:17:51,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271923 2024-12-09T08:17:51,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271923) roll requested 2024-12-09T08:17:51,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271928 2024-12-09T08:17:51,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271923 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271928 2024-12-09T08:17:51,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,934 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271928) roll requested 2024-12-09T08:17:51,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271934 2024-12-09T08:17:51,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271934 2024-12-09T08:17:51,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271934) roll requested 2024-12-09T08:17:51,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271941 2024-12-09T08:17:51,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271941 2024-12-09T08:17:51,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271941) roll requested 2024-12-09T08:17:51,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271948 2024-12-09T08:17:51,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271941 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271948 2024-12-09T08:17:51,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,955 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271948) roll requested 2024-12-09T08:17:51,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271955 2024-12-09T08:17:51,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:51,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271948 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271955 2024-12-09T08:17:51,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:51,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:51,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271955) roll requested 2024-12-09T08:17:51,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732271996 2024-12-09T08:17:52,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271955 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271996 2024-12-09T08:17:52,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732271996) roll requested 2024-12-09T08:17:52,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272005 2024-12-09T08:17:52,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732271996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272005 2024-12-09T08:17:52,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272005) roll requested 2024-12-09T08:17:52,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272011 2024-12-09T08:17:52,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272005 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272011 2024-12-09T08:17:52,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272011) roll requested 2024-12-09T08:17:52,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272023 2024-12-09T08:17:52,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272023 2024-12-09T08:17:52,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272023) roll requested 2024-12-09T08:17:52,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272030 2024-12-09T08:17:52,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272030 2024-12-09T08:17:52,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,037 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272030) roll requested 2024-12-09T08:17:52,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272037 2024-12-09T08:17:52,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272030 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272037 2024-12-09T08:17:52,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272037) roll requested 2024-12-09T08:17:52,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272049 2024-12-09T08:17:52,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272049 2024-12-09T08:17:52,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272049) roll requested 2024-12-09T08:17:52,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272055 2024-12-09T08:17:52,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272055 2024-12-09T08:17:52,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272055) roll requested 2024-12-09T08:17:52,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272061 2024-12-09T08:17:52,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272061 2024-12-09T08:17:52,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272061) roll requested 2024-12-09T08:17:52,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272068 2024-12-09T08:17:52,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272061 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272068 2024-12-09T08:17:52,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272068) roll requested 2024-12-09T08:17:52,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272076 2024-12-09T08:17:52,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272076 2024-12-09T08:17:52,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,081 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272076) roll requested 2024-12-09T08:17:52,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272081 2024-12-09T08:17:52,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272081 2024-12-09T08:17:52,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,087 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272081) roll requested 2024-12-09T08:17:52,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272087 2024-12-09T08:17:52,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272081 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272087 2024-12-09T08:17:52,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,093 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272087) roll requested 2024-12-09T08:17:52,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272093 2024-12-09T08:17:52,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272087 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272093 2024-12-09T08:17:52,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272093) roll requested 2024-12-09T08:17:52,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272100 2024-12-09T08:17:52,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272100 2024-12-09T08:17:52,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,107 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272100) roll requested 2024-12-09T08:17:52,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272108 2024-12-09T08:17:52,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272108 2024-12-09T08:17:52,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272108) roll requested 2024-12-09T08:17:52,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272117 2024-12-09T08:17:52,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272117 2024-12-09T08:17:52,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,126 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272117) roll requested 2024-12-09T08:17:52,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272127 2024-12-09T08:17:52,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272127 2024-12-09T08:17:52,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,135 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272127) roll requested 2024-12-09T08:17:52,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272136 2024-12-09T08:17:52,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272127 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272136 2024-12-09T08:17:52,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272136) roll requested 2024-12-09T08:17:52,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272145 2024-12-09T08:17:52,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272145 2024-12-09T08:17:52,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272145) roll requested 2024-12-09T08:17:52,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272150 2024-12-09T08:17:52,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272150 2024-12-09T08:17:52,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,155 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272150) roll requested 2024-12-09T08:17:52,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272155 2024-12-09T08:17:52,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272155 2024-12-09T08:17:52,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,161 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272155) roll requested 2024-12-09T08:17:52,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272161 2024-12-09T08:17:52,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272155 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272161 2024-12-09T08:17:52,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272161) roll requested 2024-12-09T08:17:52,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272171 2024-12-09T08:17:52,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272171 2024-12-09T08:17:52,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272171) roll requested 2024-12-09T08:17:52,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272179 2024-12-09T08:17:52,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272179 2024-12-09T08:17:52,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,185 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272179) roll requested 2024-12-09T08:17:52,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272185 2024-12-09T08:17:52,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272185 2024-12-09T08:17:52,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,194 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272185) roll requested 2024-12-09T08:17:52,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272194 2024-12-09T08:17:52,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272194 2024-12-09T08:17:52,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272194) roll requested 2024-12-09T08:17:52,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272201 2024-12-09T08:17:52,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272194 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272201 2024-12-09T08:17:52,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272201) roll requested 2024-12-09T08:17:52,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272208 2024-12-09T08:17:52,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272208 2024-12-09T08:17:52,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272208) roll requested 2024-12-09T08:17:52,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272214 2024-12-09T08:17:52,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272214 2024-12-09T08:17:52,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,221 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272214) roll requested 2024-12-09T08:17:52,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272222 2024-12-09T08:17:52,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272222 2024-12-09T08:17:52,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272222) roll requested 2024-12-09T08:17:52,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272229 2024-12-09T08:17:52,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272229 2024-12-09T08:17:52,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272229) roll requested 2024-12-09T08:17:52,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272237 2024-12-09T08:17:52,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272237 2024-12-09T08:17:52,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272237) roll requested 2024-12-09T08:17:52,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272249 2024-12-09T08:17:52,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272249 2024-12-09T08:17:52,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,254 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272249) roll requested 2024-12-09T08:17:52,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272254 2024-12-09T08:17:52,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272254 2024-12-09T08:17:52,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272254) roll requested 2024-12-09T08:17:52,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272259 2024-12-09T08:17:52,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272254 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272259 2024-12-09T08:17:52,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,265 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272259) roll requested 2024-12-09T08:17:52,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272266 2024-12-09T08:17:52,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272266 2024-12-09T08:17:52,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272266) roll requested 2024-12-09T08:17:52,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272273 2024-12-09T08:17:52,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272266 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272273 2024-12-09T08:17:52,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272273) roll requested 2024-12-09T08:17:52,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272281 2024-12-09T08:17:52,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272281 2024-12-09T08:17:52,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272281) roll requested 2024-12-09T08:17:52,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272288 2024-12-09T08:17:52,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272288 2024-12-09T08:17:52,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,296 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272288) roll requested 2024-12-09T08:17:52,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272297 2024-12-09T08:17:52,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272297 2024-12-09T08:17:52,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272297) roll requested 2024-12-09T08:17:52,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272304 2024-12-09T08:17:52,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272297 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272304 2024-12-09T08:17:52,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272304) roll requested 2024-12-09T08:17:52,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272312 2024-12-09T08:17:52,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272312 2024-12-09T08:17:52,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272312) roll requested 2024-12-09T08:17:52,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272321 2024-12-09T08:17:52,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272321 2024-12-09T08:17:52,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,329 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272321) roll requested 2024-12-09T08:17:52,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272330 2024-12-09T08:17:52,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272330 2024-12-09T08:17:52,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272330) roll requested 2024-12-09T08:17:52,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272339 2024-12-09T08:17:52,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272330 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272339 2024-12-09T08:17:52,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272339) roll requested 2024-12-09T08:17:52,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272348 2024-12-09T08:17:52,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272348 2024-12-09T08:17:52,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272348) roll requested 2024-12-09T08:17:52,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272356 2024-12-09T08:17:52,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272348 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272356 2024-12-09T08:17:52,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,364 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272356) roll requested 2024-12-09T08:17:52,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272365 2024-12-09T08:17:52,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272365 2024-12-09T08:17:52,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272365) roll requested 2024-12-09T08:17:52,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272377 2024-12-09T08:17:52,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272365 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272377 2024-12-09T08:17:52,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272377) roll requested 2024-12-09T08:17:52,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272384 2024-12-09T08:17:52,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272377 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272384 2024-12-09T08:17:52,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272384) roll requested 2024-12-09T08:17:52,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272389 2024-12-09T08:17:52,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272389 2024-12-09T08:17:52,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272389) roll requested 2024-12-09T08:17:52,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272394 2024-12-09T08:17:52,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272394 2024-12-09T08:17:52,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272394) roll requested 2024-12-09T08:17:52,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272400 2024-12-09T08:17:52,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272400 2024-12-09T08:17:52,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,407 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272400) roll requested 2024-12-09T08:17:52,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272407 2024-12-09T08:17:52,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272407 2024-12-09T08:17:52,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272407) roll requested 2024-12-09T08:17:52,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272415 2024-12-09T08:17:52,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,424 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272415 2024-12-09T08:17:52,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272415) roll requested 2024-12-09T08:17:52,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272425 2024-12-09T08:17:52,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272425 2024-12-09T08:17:52,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,430 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272425) roll requested 2024-12-09T08:17:52,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272430 2024-12-09T08:17:52,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272430 2024-12-09T08:17:52,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,435 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272430) roll requested 2024-12-09T08:17:52,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272435 2024-12-09T08:17:52,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272435 2024-12-09T08:17:52,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,442 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272435) roll requested 2024-12-09T08:17:52,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272442 2024-12-09T08:17:52,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272435 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272442 2024-12-09T08:17:52,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272442) roll requested 2024-12-09T08:17:52,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272449 2024-12-09T08:17:52,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272442 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272449 2024-12-09T08:17:52,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,462 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272449) roll requested 2024-12-09T08:17:52,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272462 2024-12-09T08:17:52,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272462 2024-12-09T08:17:52,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,469 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272462) roll requested 2024-12-09T08:17:52,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272469 2024-12-09T08:17:52,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272462 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272469 2024-12-09T08:17:52,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,479 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272469) roll requested 2024-12-09T08:17:52,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272479 2024-12-09T08:17:52,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272469 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272479 2024-12-09T08:17:52,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272479) roll requested 2024-12-09T08:17:52,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272487 2024-12-09T08:17:52,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272479 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272487 2024-12-09T08:17:52,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272487) roll requested 2024-12-09T08:17:52,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272492 2024-12-09T08:17:52,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272492 2024-12-09T08:17:52,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,497 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272492) roll requested 2024-12-09T08:17:52,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272497 2024-12-09T08:17:52,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272492 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272497 2024-12-09T08:17:52,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272497) roll requested 2024-12-09T08:17:52,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272503 2024-12-09T08:17:52,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272503 2024-12-09T08:17:52,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,509 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272503) roll requested 2024-12-09T08:17:52,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272509 2024-12-09T08:17:52,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272509 2024-12-09T08:17:52,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,517 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272509) roll requested 2024-12-09T08:17:52,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272517 2024-12-09T08:17:52,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272509 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272517 2024-12-09T08:17:52,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272517) roll requested 2024-12-09T08:17:52,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272526 2024-12-09T08:17:52,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272526 2024-12-09T08:17:52,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272526) roll requested 2024-12-09T08:17:52,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272535 2024-12-09T08:17:52,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272535 2024-12-09T08:17:52,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272535) roll requested 2024-12-09T08:17:52,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272543 2024-12-09T08:17:52,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272543 2024-12-09T08:17:52,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,551 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272543) roll requested 2024-12-09T08:17:52,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272551 2024-12-09T08:17:52,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272551 2024-12-09T08:17:52,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272551) roll requested 2024-12-09T08:17:52,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272560 2024-12-09T08:17:52,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272551 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272560 2024-12-09T08:17:52,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272560) roll requested 2024-12-09T08:17:52,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272569 2024-12-09T08:17:52,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272569 2024-12-09T08:17:52,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272569) roll requested 2024-12-09T08:17:52,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272577 2024-12-09T08:17:52,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272577 2024-12-09T08:17:52,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272577) roll requested 2024-12-09T08:17:52,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272583 2024-12-09T08:17:52,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272583 2024-12-09T08:17:52,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272583) roll requested 2024-12-09T08:17:52,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272589 2024-12-09T08:17:52,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272583 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272589 2024-12-09T08:17:52,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,594 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272589) roll requested 2024-12-09T08:17:52,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272595 2024-12-09T08:17:52,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272595 2024-12-09T08:17:52,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272595) roll requested 2024-12-09T08:17:52,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272601 2024-12-09T08:17:52,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272601 2024-12-09T08:17:52,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,607 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272601) roll requested 2024-12-09T08:17:52,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272607 2024-12-09T08:17:52,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272607 2024-12-09T08:17:52,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,621 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272607) roll requested 2024-12-09T08:17:52,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272621 2024-12-09T08:17:52,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272607 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272621 2024-12-09T08:17:52,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272621) roll requested 2024-12-09T08:17:52,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272631 2024-12-09T08:17:52,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272631 2024-12-09T08:17:52,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272631) roll requested 2024-12-09T08:17:52,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272637 2024-12-09T08:17:52,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272637 2024-12-09T08:17:52,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272637) roll requested 2024-12-09T08:17:52,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272645 2024-12-09T08:17:52,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272637 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272645 2024-12-09T08:17:52,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272645) roll requested 2024-12-09T08:17:52,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272651 2024-12-09T08:17:52,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272651 2024-12-09T08:17:52,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272651) roll requested 2024-12-09T08:17:52,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272657 2024-12-09T08:17:52,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272657 2024-12-09T08:17:52,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,661 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272657) roll requested 2024-12-09T08:17:52,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272662 2024-12-09T08:17:52,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272662 2024-12-09T08:17:52,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272662) roll requested 2024-12-09T08:17:52,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272668 2024-12-09T08:17:52,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272668 2024-12-09T08:17:52,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272668) roll requested 2024-12-09T08:17:52,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272673 2024-12-09T08:17:52,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272673 2024-12-09T08:17:52,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272673) roll requested 2024-12-09T08:17:52,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272685 2024-12-09T08:17:52,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272685 2024-12-09T08:17:52,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272685) roll requested 2024-12-09T08:17:52,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272701 2024-12-09T08:17:52,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272701 2024-12-09T08:17:52,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272701) roll requested 2024-12-09T08:17:52,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272709 2024-12-09T08:17:52,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272709 2024-12-09T08:17:52,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272709) roll requested 2024-12-09T08:17:52,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272715 2024-12-09T08:17:52,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272715 2024-12-09T08:17:52,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272715) roll requested 2024-12-09T08:17:52,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272720 2024-12-09T08:17:52,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272720 2024-12-09T08:17:52,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272720) roll requested 2024-12-09T08:17:52,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272726 2024-12-09T08:17:52,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272726 2024-12-09T08:17:52,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272726) roll requested 2024-12-09T08:17:52,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272732 2024-12-09T08:17:52,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272732 2024-12-09T08:17:52,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272732) roll requested 2024-12-09T08:17:52,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272739 2024-12-09T08:17:52,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272739 2024-12-09T08:17:52,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,746 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272739) roll requested 2024-12-09T08:17:52,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272746 2024-12-09T08:17:52,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272739 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272746 2024-12-09T08:17:52,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,755 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272746) roll requested 2024-12-09T08:17:52,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272755 2024-12-09T08:17:52,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272755 2024-12-09T08:17:52,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272755) roll requested 2024-12-09T08:17:52,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272761 2024-12-09T08:17:52,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272755 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272761 2024-12-09T08:17:52,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,766 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272761) roll requested 2024-12-09T08:17:52,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272766 2024-12-09T08:17:52,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272766 2024-12-09T08:17:52,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272766) roll requested 2024-12-09T08:17:52,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272772 2024-12-09T08:17:52,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272766 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272772 2024-12-09T08:17:52,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,781 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272772) roll requested 2024-12-09T08:17:52,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272781 2024-12-09T08:17:52,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272781 2024-12-09T08:17:52,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272781) roll requested 2024-12-09T08:17:52,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272791 2024-12-09T08:17:52,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272781 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272791 2024-12-09T08:17:52,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272791) roll requested 2024-12-09T08:17:52,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272800 2024-12-09T08:17:52,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272800 2024-12-09T08:17:52,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,807 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272800) roll requested 2024-12-09T08:17:52,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272808 2024-12-09T08:17:52,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272808 2024-12-09T08:17:52,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,814 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272808) roll requested 2024-12-09T08:17:52,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272814 2024-12-09T08:17:52,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272814 2024-12-09T08:17:52,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272814) roll requested 2024-12-09T08:17:52,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272822 2024-12-09T08:17:52,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272822 2024-12-09T08:17:52,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,830 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272822) roll requested 2024-12-09T08:17:52,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272830 2024-12-09T08:17:52,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272830 2024-12-09T08:17:52,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272830) roll requested 2024-12-09T08:17:52,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272837 2024-12-09T08:17:52,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272830 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272837 2024-12-09T08:17:52,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272837) roll requested 2024-12-09T08:17:52,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272849 2024-12-09T08:17:52,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272849 2024-12-09T08:17:52,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272849) roll requested 2024-12-09T08:17:52,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272859 2024-12-09T08:17:52,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272859 2024-12-09T08:17:52,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272859) roll requested 2024-12-09T08:17:52,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272867 2024-12-09T08:17:52,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272859 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272867 2024-12-09T08:17:52,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272867) roll requested 2024-12-09T08:17:52,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272877 2024-12-09T08:17:52,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272877 2024-12-09T08:17:52,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,884 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272877) roll requested 2024-12-09T08:17:52,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272885 2024-12-09T08:17:52,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272877 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272885 2024-12-09T08:17:52,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272885) roll requested 2024-12-09T08:17:52,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272893 2024-12-09T08:17:52,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272893 2024-12-09T08:17:52,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272893) roll requested 2024-12-09T08:17:52,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272899 2024-12-09T08:17:52,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272899 2024-12-09T08:17:52,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272899) roll requested 2024-12-09T08:17:52,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272904 2024-12-09T08:17:52,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272899 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272904 2024-12-09T08:17:52,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272904) roll requested 2024-12-09T08:17:52,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272912 2024-12-09T08:17:52,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272904 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272912 2024-12-09T08:17:52,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272912) roll requested 2024-12-09T08:17:52,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272920 2024-12-09T08:17:52,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272920 2024-12-09T08:17:52,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272920) roll requested 2024-12-09T08:17:52,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272925 2024-12-09T08:17:52,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272925 2024-12-09T08:17:52,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,930 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272925) roll requested 2024-12-09T08:17:52,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272930 2024-12-09T08:17:52,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272930 2024-12-09T08:17:52,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272930) roll requested 2024-12-09T08:17:52,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272938 2024-12-09T08:17:52,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272930 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272938 2024-12-09T08:17:52,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272938) roll requested 2024-12-09T08:17:52,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272945 2024-12-09T08:17:52,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272938 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272945 2024-12-09T08:17:52,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,954 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272945) roll requested 2024-12-09T08:17:52,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272955 2024-12-09T08:17:52,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272955 2024-12-09T08:17:52,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272955) roll requested 2024-12-09T08:17:52,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272965 2024-12-09T08:17:52,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272955 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272965 2024-12-09T08:17:52,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272965) roll requested 2024-12-09T08:17:52,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272971 2024-12-09T08:17:52,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272971 2024-12-09T08:17:52,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272971) roll requested 2024-12-09T08:17:52,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272976 2024-12-09T08:17:52,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272971 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272976 2024-12-09T08:17:52,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272976) roll requested 2024-12-09T08:17:52,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272982 2024-12-09T08:17:52,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:52,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272982 2024-12-09T08:17:52,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:52,993 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:52,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272982) roll requested 2024-12-09T08:17:52,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732272993 2024-12-09T08:17:53,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272993 2024-12-09T08:17:53,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,002 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732272993) roll requested 2024-12-09T08:17:53,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273002 2024-12-09T08:17:53,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732272993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273002 2024-12-09T08:17:53,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273002) roll requested 2024-12-09T08:17:53,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273016 2024-12-09T08:17:53,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273002 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273016 2024-12-09T08:17:53,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273016) roll requested 2024-12-09T08:17:53,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273022 2024-12-09T08:17:53,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273022 2024-12-09T08:17:53,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273022) roll requested 2024-12-09T08:17:53,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273027 2024-12-09T08:17:53,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273022 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273027 2024-12-09T08:17:53,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273027) roll requested 2024-12-09T08:17:53,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273033 2024-12-09T08:17:53,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273027 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273033 2024-12-09T08:17:53,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273033) roll requested 2024-12-09T08:17:53,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273041 2024-12-09T08:17:53,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273041 2024-12-09T08:17:53,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,051 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273041) roll requested 2024-12-09T08:17:53,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273051 2024-12-09T08:17:53,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273051 2024-12-09T08:17:53,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,057 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273051) roll requested 2024-12-09T08:17:53,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273057 2024-12-09T08:17:53,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273051 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273057 2024-12-09T08:17:53,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273057) roll requested 2024-12-09T08:17:53,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273064 2024-12-09T08:17:53,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273064 2024-12-09T08:17:53,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273064) roll requested 2024-12-09T08:17:53,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273070 2024-12-09T08:17:53,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273070 2024-12-09T08:17:53,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273070) roll requested 2024-12-09T08:17:53,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273076 2024-12-09T08:17:53,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273076 2024-12-09T08:17:53,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273076) roll requested 2024-12-09T08:17:53,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273082 2024-12-09T08:17:53,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273082 2024-12-09T08:17:53,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,091 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273082) roll requested 2024-12-09T08:17:53,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273092 2024-12-09T08:17:53,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273092 2024-12-09T08:17:53,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273092) roll requested 2024-12-09T08:17:53,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273100 2024-12-09T08:17:53,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273092 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273100 2024-12-09T08:17:53,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273100) roll requested 2024-12-09T08:17:53,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273106 2024-12-09T08:17:53,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273106 2024-12-09T08:17:53,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,111 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273106) roll requested 2024-12-09T08:17:53,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273111 2024-12-09T08:17:53,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273111 2024-12-09T08:17:53,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273111) roll requested 2024-12-09T08:17:53,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273116 2024-12-09T08:17:53,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273111 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273116 2024-12-09T08:17:53,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273116) roll requested 2024-12-09T08:17:53,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273122 2024-12-09T08:17:53,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273122 2024-12-09T08:17:53,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273122) roll requested 2024-12-09T08:17:53,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273129 2024-12-09T08:17:53,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273122 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273129 2024-12-09T08:17:53,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,135 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273129) roll requested 2024-12-09T08:17:53,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273136 2024-12-09T08:17:53,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273136 2024-12-09T08:17:53,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273136) roll requested 2024-12-09T08:17:53,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273144 2024-12-09T08:17:53,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273144 2024-12-09T08:17:53,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273144) roll requested 2024-12-09T08:17:53,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273151 2024-12-09T08:17:53,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273144 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273151 2024-12-09T08:17:53,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273151) roll requested 2024-12-09T08:17:53,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273158 2024-12-09T08:17:53,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273158 2024-12-09T08:17:53,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,163 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273158) roll requested 2024-12-09T08:17:53,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273163 2024-12-09T08:17:53,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273163 2024-12-09T08:17:53,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,167 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273163) roll requested 2024-12-09T08:17:53,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273168 2024-12-09T08:17:53,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273163 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273168 2024-12-09T08:17:53,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,173 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273168) roll requested 2024-12-09T08:17:53,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273173 2024-12-09T08:17:53,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273173 2024-12-09T08:17:53,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,178 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273173) roll requested 2024-12-09T08:17:53,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273178 2024-12-09T08:17:53,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273178 2024-12-09T08:17:53,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,185 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273178) roll requested 2024-12-09T08:17:53,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273185 2024-12-09T08:17:53,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273178 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273185 2024-12-09T08:17:53,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,193 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273185) roll requested 2024-12-09T08:17:53,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273193 2024-12-09T08:17:53,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273193 2024-12-09T08:17:53,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,199 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273193) roll requested 2024-12-09T08:17:53,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273200 2024-12-09T08:17:53,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273200 2024-12-09T08:17:53,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273200) roll requested 2024-12-09T08:17:53,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273206 2024-12-09T08:17:53,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273200 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273206 2024-12-09T08:17:53,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,213 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273206) roll requested 2024-12-09T08:17:53,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273214 2024-12-09T08:17:53,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273206 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273214 2024-12-09T08:17:53,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273214) roll requested 2024-12-09T08:17:53,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273222 2024-12-09T08:17:53,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273222 2024-12-09T08:17:53,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273222) roll requested 2024-12-09T08:17:53,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273230 2024-12-09T08:17:53,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273230 2024-12-09T08:17:53,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273230) roll requested 2024-12-09T08:17:53,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273237 2024-12-09T08:17:53,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273237 2024-12-09T08:17:53,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273237) roll requested 2024-12-09T08:17:53,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273244 2024-12-09T08:17:53,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273244 2024-12-09T08:17:53,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273244) roll requested 2024-12-09T08:17:53,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273252 2024-12-09T08:17:53,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273252 2024-12-09T08:17:53,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273252) roll requested 2024-12-09T08:17:53,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273261 2024-12-09T08:17:53,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273261 2024-12-09T08:17:53,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273261) roll requested 2024-12-09T08:17:53,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273271 2024-12-09T08:17:53,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273271 2024-12-09T08:17:53,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273271) roll requested 2024-12-09T08:17:53,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273281 2024-12-09T08:17:53,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273281 2024-12-09T08:17:53,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,290 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273281) roll requested 2024-12-09T08:17:53,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273290 2024-12-09T08:17:53,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273290 2024-12-09T08:17:53,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273290) roll requested 2024-12-09T08:17:53,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273299 2024-12-09T08:17:53,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273290 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273299 2024-12-09T08:17:53,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273299) roll requested 2024-12-09T08:17:53,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273309 2024-12-09T08:17:53,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273309 2024-12-09T08:17:53,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273309) roll requested 2024-12-09T08:17:53,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273317 2024-12-09T08:17:53,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273309 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273317 2024-12-09T08:17:53,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,325 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273317) roll requested 2024-12-09T08:17:53,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273325 2024-12-09T08:17:53,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273325 2024-12-09T08:17:53,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,333 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273325) roll requested 2024-12-09T08:17:53,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273333 2024-12-09T08:17:53,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273333 2024-12-09T08:17:53,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273333) roll requested 2024-12-09T08:17:53,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273345 2024-12-09T08:17:53,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273345 2024-12-09T08:17:53,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273345) roll requested 2024-12-09T08:17:53,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273350 2024-12-09T08:17:53,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273350 2024-12-09T08:17:53,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273350) roll requested 2024-12-09T08:17:53,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273356 2024-12-09T08:17:53,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273356 2024-12-09T08:17:53,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273356) roll requested 2024-12-09T08:17:53,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273362 2024-12-09T08:17:53,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273362 2024-12-09T08:17:53,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273362) roll requested 2024-12-09T08:17:53,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273369 2024-12-09T08:17:53,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273369 2024-12-09T08:17:53,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273369) roll requested 2024-12-09T08:17:53,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273375 2024-12-09T08:17:53,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273375 2024-12-09T08:17:53,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,380 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273375) roll requested 2024-12-09T08:17:53,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273380 2024-12-09T08:17:53,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273375 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273380 2024-12-09T08:17:53,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,385 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273380) roll requested 2024-12-09T08:17:53,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273385 2024-12-09T08:17:53,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273385 2024-12-09T08:17:53,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,390 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273385) roll requested 2024-12-09T08:17:53,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273390 2024-12-09T08:17:53,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273390 2024-12-09T08:17:53,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,395 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273390) roll requested 2024-12-09T08:17:53,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273395 2024-12-09T08:17:53,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273395 2024-12-09T08:17:53,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,401 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273395) roll requested 2024-12-09T08:17:53,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273401 2024-12-09T08:17:53,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,408 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273395 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273401 2024-12-09T08:17:53,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,409 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273401) roll requested 2024-12-09T08:17:53,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273409 2024-12-09T08:17:53,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273409 2024-12-09T08:17:53,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273409) roll requested 2024-12-09T08:17:53,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273415 2024-12-09T08:17:53,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273409 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273415 2024-12-09T08:17:53,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273415) roll requested 2024-12-09T08:17:53,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273420 2024-12-09T08:17:53,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273420 2024-12-09T08:17:53,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,425 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273420) roll requested 2024-12-09T08:17:53,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273425 2024-12-09T08:17:53,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273420 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273425 2024-12-09T08:17:53,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,430 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273425) roll requested 2024-12-09T08:17:53,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273430 2024-12-09T08:17:53,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273430 2024-12-09T08:17:53,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,435 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273430) roll requested 2024-12-09T08:17:53,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273435 2024-12-09T08:17:53,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273435 2024-12-09T08:17:53,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273435) roll requested 2024-12-09T08:17:53,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273440 2024-12-09T08:17:53,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273435 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273440 2024-12-09T08:17:53,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273440) roll requested 2024-12-09T08:17:53,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273445 2024-12-09T08:17:53,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273445 2024-12-09T08:17:53,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273445) roll requested 2024-12-09T08:17:53,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273451 2024-12-09T08:17:53,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273451 2024-12-09T08:17:53,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273451) roll requested 2024-12-09T08:17:53,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273457 2024-12-09T08:17:53,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273451 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273457 2024-12-09T08:17:53,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273457) roll requested 2024-12-09T08:17:53,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273464 2024-12-09T08:17:53,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273464 2024-12-09T08:17:53,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273464) roll requested 2024-12-09T08:17:53,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273471 2024-12-09T08:17:53,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273464 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273471 2024-12-09T08:17:53,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273471) roll requested 2024-12-09T08:17:53,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273478 2024-12-09T08:17:53,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273478 2024-12-09T08:17:53,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273478) roll requested 2024-12-09T08:17:53,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273487 2024-12-09T08:17:53,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273487 2024-12-09T08:17:53,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,494 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273487) roll requested 2024-12-09T08:17:53,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273495 2024-12-09T08:17:53,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273495 2024-12-09T08:17:53,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273495) roll requested 2024-12-09T08:17:53,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273504 2024-12-09T08:17:53,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273495 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273504 2024-12-09T08:17:53,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273504) roll requested 2024-12-09T08:17:53,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273512 2024-12-09T08:17:53,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273504 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273512 2024-12-09T08:17:53,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273512) roll requested 2024-12-09T08:17:53,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273519 2024-12-09T08:17:53,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273519 2024-12-09T08:17:53,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273519) roll requested 2024-12-09T08:17:53,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273527 2024-12-09T08:17:53,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273527 2024-12-09T08:17:53,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273527) roll requested 2024-12-09T08:17:53,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273535 2024-12-09T08:17:53,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273535 2024-12-09T08:17:53,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273535) roll requested 2024-12-09T08:17:53,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273543 2024-12-09T08:17:53,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273543 2024-12-09T08:17:53,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273543) roll requested 2024-12-09T08:17:53,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273552 2024-12-09T08:17:53,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273552 2024-12-09T08:17:53,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,562 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273552) roll requested 2024-12-09T08:17:53,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273562 2024-12-09T08:17:53,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273562 2024-12-09T08:17:53,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,575 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273562) roll requested 2024-12-09T08:17:53,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273575 2024-12-09T08:17:53,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273562 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273575 2024-12-09T08:17:53,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,582 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273575) roll requested 2024-12-09T08:17:53,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273582 2024-12-09T08:17:53,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273575 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273582 2024-12-09T08:17:53,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273582) roll requested 2024-12-09T08:17:53,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273587 2024-12-09T08:17:53,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273582 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273587 2024-12-09T08:17:53,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273587) roll requested 2024-12-09T08:17:53,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273591 2024-12-09T08:17:53,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273591 2024-12-09T08:17:53,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273591) roll requested 2024-12-09T08:17:53,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273598 2024-12-09T08:17:53,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273591 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273598 2024-12-09T08:17:53,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273598) roll requested 2024-12-09T08:17:53,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273605 2024-12-09T08:17:53,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273598 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273605 2024-12-09T08:17:53,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,615 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273605) roll requested 2024-12-09T08:17:53,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273615 2024-12-09T08:17:53,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273615 2024-12-09T08:17:53,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273615) roll requested 2024-12-09T08:17:53,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273624 2024-12-09T08:17:53,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273615 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273624 2024-12-09T08:17:53,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273624) roll requested 2024-12-09T08:17:53,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273631 2024-12-09T08:17:53,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273624 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273631 2024-12-09T08:17:53,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273631) roll requested 2024-12-09T08:17:53,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273640 2024-12-09T08:17:53,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273640 2024-12-09T08:17:53,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273640) roll requested 2024-12-09T08:17:53,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273649 2024-12-09T08:17:53,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273640 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273649 2024-12-09T08:17:53,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273649) roll requested 2024-12-09T08:17:53,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273659 2024-12-09T08:17:53,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273659 2024-12-09T08:17:53,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273659) roll requested 2024-12-09T08:17:53,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273667 2024-12-09T08:17:53,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273659 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273667 2024-12-09T08:17:53,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273667) roll requested 2024-12-09T08:17:53,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273672 2024-12-09T08:17:53,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273672 2024-12-09T08:17:53,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273672) roll requested 2024-12-09T08:17:53,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273676 2024-12-09T08:17:53,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273676 2024-12-09T08:17:53,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,682 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273676) roll requested 2024-12-09T08:17:53,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273682 2024-12-09T08:17:53,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273682 2024-12-09T08:17:53,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273682) roll requested 2024-12-09T08:17:53,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273689 2024-12-09T08:17:53,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273682 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273689 2024-12-09T08:17:53,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273689) roll requested 2024-12-09T08:17:53,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273697 2024-12-09T08:17:53,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273697 2024-12-09T08:17:53,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273697) roll requested 2024-12-09T08:17:53,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273705 2024-12-09T08:17:53,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273705 2024-12-09T08:17:53,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,715 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273705) roll requested 2024-12-09T08:17:53,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273715 2024-12-09T08:17:53,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273715 2024-12-09T08:17:53,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273715) roll requested 2024-12-09T08:17:53,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273724 2024-12-09T08:17:53,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273724 2024-12-09T08:17:53,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273724) roll requested 2024-12-09T08:17:53,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273733 2024-12-09T08:17:53,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273724 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273733 2024-12-09T08:17:53,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,743 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273733) roll requested 2024-12-09T08:17:53,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273743 2024-12-09T08:17:53,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273743 2024-12-09T08:17:53,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273743) roll requested 2024-12-09T08:17:53,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273752 2024-12-09T08:17:53,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273743 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273752 2024-12-09T08:17:53,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273752) roll requested 2024-12-09T08:17:53,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273759 2024-12-09T08:17:53,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273752 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273759 2024-12-09T08:17:53,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,767 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273759) roll requested 2024-12-09T08:17:53,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273767 2024-12-09T08:17:53,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273767 2024-12-09T08:17:53,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273767) roll requested 2024-12-09T08:17:53,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273775 2024-12-09T08:17:53,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273775 2024-12-09T08:17:53,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273775) roll requested 2024-12-09T08:17:53,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273783 2024-12-09T08:17:53,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273783 2024-12-09T08:17:53,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273783) roll requested 2024-12-09T08:17:53,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273791 2024-12-09T08:17:53,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273791 2024-12-09T08:17:53,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,797 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273791) roll requested 2024-12-09T08:17:53,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273797 2024-12-09T08:17:53,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273797 2024-12-09T08:17:53,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273797) roll requested 2024-12-09T08:17:53,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273804 2024-12-09T08:17:53,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273804 2024-12-09T08:17:53,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273804) roll requested 2024-12-09T08:17:53,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273812 2024-12-09T08:17:53,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273812 2024-12-09T08:17:53,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273812) roll requested 2024-12-09T08:17:53,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273822 2024-12-09T08:17:53,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273822 2024-12-09T08:17:53,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273822) roll requested 2024-12-09T08:17:53,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273830 2024-12-09T08:17:53,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273830 2024-12-09T08:17:53,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273830) roll requested 2024-12-09T08:17:53,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273837 2024-12-09T08:17:53,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273830 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273837 2024-12-09T08:17:53,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,844 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273837) roll requested 2024-12-09T08:17:53,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273844 2024-12-09T08:17:53,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273844 2024-12-09T08:17:53,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273844) roll requested 2024-12-09T08:17:53,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273853 2024-12-09T08:17:53,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273853 2024-12-09T08:17:53,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273853) roll requested 2024-12-09T08:17:53,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273860 2024-12-09T08:17:53,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273860 2024-12-09T08:17:53,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273860) roll requested 2024-12-09T08:17:53,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273868 2024-12-09T08:17:53,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273868 2024-12-09T08:17:53,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273868) roll requested 2024-12-09T08:17:53,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273877 2024-12-09T08:17:53,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273877 2024-12-09T08:17:53,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,884 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273877) roll requested 2024-12-09T08:17:53,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273884 2024-12-09T08:17:53,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273877 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273884 2024-12-09T08:17:53,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,892 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273884) roll requested 2024-12-09T08:17:53,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273893 2024-12-09T08:17:53,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273893 2024-12-09T08:17:53,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273893) roll requested 2024-12-09T08:17:53,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273900 2024-12-09T08:17:53,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273900 2024-12-09T08:17:53,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,909 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273900) roll requested 2024-12-09T08:17:53,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273909 2024-12-09T08:17:53,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273909 2024-12-09T08:17:53,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273909) roll requested 2024-12-09T08:17:53,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273918 2024-12-09T08:17:53,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273918 2024-12-09T08:17:53,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273918) roll requested 2024-12-09T08:17:53,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273925 2024-12-09T08:17:53,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273918 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273925 2024-12-09T08:17:53,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273925) roll requested 2024-12-09T08:17:53,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273932 2024-12-09T08:17:53,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273932 2024-12-09T08:17:53,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273932) roll requested 2024-12-09T08:17:53,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273940 2024-12-09T08:17:53,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273940 2024-12-09T08:17:53,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,949 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273940) roll requested 2024-12-09T08:17:53,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273950 2024-12-09T08:17:53,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273950 2024-12-09T08:17:53,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,962 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273950) roll requested 2024-12-09T08:17:53,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273962 2024-12-09T08:17:53,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273950 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273962 2024-12-09T08:17:53,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273962) roll requested 2024-12-09T08:17:53,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273969 2024-12-09T08:17:53,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273962 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273969 2024-12-09T08:17:53,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273969) roll requested 2024-12-09T08:17:53,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273977 2024-12-09T08:17:53,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273977 2024-12-09T08:17:53,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273977) roll requested 2024-12-09T08:17:53,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273986 2024-12-09T08:17:53,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273986 2024-12-09T08:17:53,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,990 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273986) roll requested 2024-12-09T08:17:53,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273991 2024-12-09T08:17:53,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273986 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273991 2024-12-09T08:17:53,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:53,995 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:53,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273991) roll requested 2024-12-09T08:17:53,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732273995 2024-12-09T08:17:53,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:53,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273995 2024-12-09T08:17:54,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732273995) roll requested 2024-12-09T08:17:54,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274000 2024-12-09T08:17:54,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732273995 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274000 2024-12-09T08:17:54,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274000) roll requested 2024-12-09T08:17:54,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274006 2024-12-09T08:17:54,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274006 2024-12-09T08:17:54,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274006) roll requested 2024-12-09T08:17:54,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274012 2024-12-09T08:17:54,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274012 2024-12-09T08:17:54,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,019 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274012) roll requested 2024-12-09T08:17:54,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274019 2024-12-09T08:17:54,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274019 2024-12-09T08:17:54,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,029 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274019) roll requested 2024-12-09T08:17:54,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274029 2024-12-09T08:17:54,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274019 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274029 2024-12-09T08:17:54,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,034 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274029) roll requested 2024-12-09T08:17:54,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274034 2024-12-09T08:17:54,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274034 2024-12-09T08:17:54,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,039 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274034) roll requested 2024-12-09T08:17:54,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274039 2024-12-09T08:17:54,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274039 2024-12-09T08:17:54,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,044 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274039) roll requested 2024-12-09T08:17:54,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274045 2024-12-09T08:17:54,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274039 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274045 2024-12-09T08:17:54,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,051 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274045) roll requested 2024-12-09T08:17:54,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274051 2024-12-09T08:17:54,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274045 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274051 2024-12-09T08:17:54,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274051) roll requested 2024-12-09T08:17:54,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274059 2024-12-09T08:17:54,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274051 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274059 2024-12-09T08:17:54,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,067 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274059) roll requested 2024-12-09T08:17:54,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274068 2024-12-09T08:17:54,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274059 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274068 2024-12-09T08:17:54,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274068) roll requested 2024-12-09T08:17:54,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274074 2024-12-09T08:17:54,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274074 2024-12-09T08:17:54,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274074) roll requested 2024-12-09T08:17:54,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274080 2024-12-09T08:17:54,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274074 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274080 2024-12-09T08:17:54,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,087 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274080) roll requested 2024-12-09T08:17:54,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274088 2024-12-09T08:17:54,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274088 2024-12-09T08:17:54,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,095 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274088) roll requested 2024-12-09T08:17:54,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274096 2024-12-09T08:17:54,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274088 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274096 2024-12-09T08:17:54,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274096) roll requested 2024-12-09T08:17:54,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274106 2024-12-09T08:17:54,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274106 2024-12-09T08:17:54,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,114 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274106) roll requested 2024-12-09T08:17:54,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274115 2024-12-09T08:17:54,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274115 2024-12-09T08:17:54,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274115) roll requested 2024-12-09T08:17:54,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274123 2024-12-09T08:17:54,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274115 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274123 2024-12-09T08:17:54,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274123) roll requested 2024-12-09T08:17:54,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274130 2024-12-09T08:17:54,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274123 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274130 2024-12-09T08:17:54,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,138 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274130) roll requested 2024-12-09T08:17:54,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274138 2024-12-09T08:17:54,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274130 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274138 2024-12-09T08:17:54,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,146 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274138) roll requested 2024-12-09T08:17:54,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274147 2024-12-09T08:17:54,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274138 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274147 2024-12-09T08:17:54,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274147) roll requested 2024-12-09T08:17:54,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274156 2024-12-09T08:17:54,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274147 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274156 2024-12-09T08:17:54,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274156) roll requested 2024-12-09T08:17:54,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274165 2024-12-09T08:17:54,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274156 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274165 2024-12-09T08:17:54,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274165) roll requested 2024-12-09T08:17:54,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274172 2024-12-09T08:17:54,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274172 2024-12-09T08:17:54,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274172) roll requested 2024-12-09T08:17:54,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274180 2024-12-09T08:17:54,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274180 2024-12-09T08:17:54,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274180) roll requested 2024-12-09T08:17:54,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274184 2024-12-09T08:17:54,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274184 2024-12-09T08:17:54,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274184) roll requested 2024-12-09T08:17:54,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274189 2024-12-09T08:17:54,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274189 2024-12-09T08:17:54,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,194 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274189) roll requested 2024-12-09T08:17:54,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274194 2024-12-09T08:17:54,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274194 2024-12-09T08:17:54,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,200 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274194) roll requested 2024-12-09T08:17:54,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274200 2024-12-09T08:17:54,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274194 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274200 2024-12-09T08:17:54,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,207 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274200) roll requested 2024-12-09T08:17:54,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274208 2024-12-09T08:17:54,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274200 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274208 2024-12-09T08:17:54,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274208) roll requested 2024-12-09T08:17:54,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274216 2024-12-09T08:17:54,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274216 2024-12-09T08:17:54,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,223 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274216) roll requested 2024-12-09T08:17:54,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274223 2024-12-09T08:17:54,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274223 2024-12-09T08:17:54,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274223) roll requested 2024-12-09T08:17:54,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274232 2024-12-09T08:17:54,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274223 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274232 2024-12-09T08:17:54,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,241 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274232) roll requested 2024-12-09T08:17:54,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274241 2024-12-09T08:17:54,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274241 2024-12-09T08:17:54,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,247 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274241) roll requested 2024-12-09T08:17:54,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274247 2024-12-09T08:17:54,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274241 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274247 2024-12-09T08:17:54,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,253 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274247) roll requested 2024-12-09T08:17:54,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274254 2024-12-09T08:17:54,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274247 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274254 2024-12-09T08:17:54,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274254) roll requested 2024-12-09T08:17:54,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274262 2024-12-09T08:17:54,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274254 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274262 2024-12-09T08:17:54,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274262) roll requested 2024-12-09T08:17:54,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274270 2024-12-09T08:17:54,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274262 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274270 2024-12-09T08:17:54,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,278 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274270) roll requested 2024-12-09T08:17:54,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274278 2024-12-09T08:17:54,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274270 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274278 2024-12-09T08:17:54,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274278) roll requested 2024-12-09T08:17:54,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274287 2024-12-09T08:17:54,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274287 2024-12-09T08:17:54,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,296 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274287) roll requested 2024-12-09T08:17:54,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274296 2024-12-09T08:17:54,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274296 2024-12-09T08:17:54,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,302 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274296) roll requested 2024-12-09T08:17:54,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274302 2024-12-09T08:17:54,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274296 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274302 2024-12-09T08:17:54,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274302) roll requested 2024-12-09T08:17:54,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274308 2024-12-09T08:17:54,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274302 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274308 2024-12-09T08:17:54,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274308) roll requested 2024-12-09T08:17:54,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274314 2024-12-09T08:17:54,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274314 2024-12-09T08:17:54,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,321 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274314) roll requested 2024-12-09T08:17:54,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274322 2024-12-09T08:17:54,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274322 2024-12-09T08:17:54,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,326 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274322) roll requested 2024-12-09T08:17:54,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274326 2024-12-09T08:17:54,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274322 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274326 2024-12-09T08:17:54,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274326) roll requested 2024-12-09T08:17:54,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274332 2024-12-09T08:17:54,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274326 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274332 2024-12-09T08:17:54,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274332) roll requested 2024-12-09T08:17:54,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274341 2024-12-09T08:17:54,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274341 2024-12-09T08:17:54,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274341) roll requested 2024-12-09T08:17:54,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274349 2024-12-09T08:17:54,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274349 2024-12-09T08:17:54,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274349) roll requested 2024-12-09T08:17:54,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274354 2024-12-09T08:17:54,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274349 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274354 2024-12-09T08:17:54,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274354) roll requested 2024-12-09T08:17:54,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274360 2024-12-09T08:17:54,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274354 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274360 2024-12-09T08:17:54,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,365 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274360) roll requested 2024-12-09T08:17:54,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274366 2024-12-09T08:17:54,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274360 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274366 2024-12-09T08:17:54,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274366) roll requested 2024-12-09T08:17:54,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274374 2024-12-09T08:17:54,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274366 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274374 2024-12-09T08:17:54,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274374) roll requested 2024-12-09T08:17:54,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274382 2024-12-09T08:17:54,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274382 2024-12-09T08:17:54,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,395 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274382) roll requested 2024-12-09T08:17:54,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274395 2024-12-09T08:17:54,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274395 2024-12-09T08:17:54,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,401 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274395) roll requested 2024-12-09T08:17:54,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274401 2024-12-09T08:17:54,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274395 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274401 2024-12-09T08:17:54,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274401) roll requested 2024-12-09T08:17:54,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274406 2024-12-09T08:17:54,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274406 2024-12-09T08:17:54,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274406) roll requested 2024-12-09T08:17:54,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274414 2024-12-09T08:17:54,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274414 2024-12-09T08:17:54,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274414) roll requested 2024-12-09T08:17:54,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274429 2024-12-09T08:17:54,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274429 2024-12-09T08:17:54,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274429) roll requested 2024-12-09T08:17:54,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274435 2024-12-09T08:17:54,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274429 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274435 2024-12-09T08:17:54,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274435) roll requested 2024-12-09T08:17:54,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274440 2024-12-09T08:17:54,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274435 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274440 2024-12-09T08:17:54,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274440) roll requested 2024-12-09T08:17:54,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274445 2024-12-09T08:17:54,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274445 2024-12-09T08:17:54,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274445) roll requested 2024-12-09T08:17:54,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274450 2024-12-09T08:17:54,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274450 2024-12-09T08:17:54,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,455 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274450) roll requested 2024-12-09T08:17:54,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274455 2024-12-09T08:17:54,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274455 2024-12-09T08:17:54,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274455) roll requested 2024-12-09T08:17:54,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274461 2024-12-09T08:17:54,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274455 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274461 2024-12-09T08:17:54,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274461) roll requested 2024-12-09T08:17:54,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274466 2024-12-09T08:17:54,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274466 2024-12-09T08:17:54,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274466) roll requested 2024-12-09T08:17:54,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274471 2024-12-09T08:17:54,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274471 2024-12-09T08:17:54,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274471) roll requested 2024-12-09T08:17:54,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274477 2024-12-09T08:17:54,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274477 2024-12-09T08:17:54,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274477) roll requested 2024-12-09T08:17:54,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274486 2024-12-09T08:17:54,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274477 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274486 2024-12-09T08:17:54,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,491 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274486) roll requested 2024-12-09T08:17:54,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274491 2024-12-09T08:17:54,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274486 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274491 2024-12-09T08:17:54,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274491) roll requested 2024-12-09T08:17:54,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274496 2024-12-09T08:17:54,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274496 2024-12-09T08:17:54,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,505 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274496) roll requested 2024-12-09T08:17:54,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274505 2024-12-09T08:17:54,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274505 2024-12-09T08:17:54,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,510 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274505) roll requested 2024-12-09T08:17:54,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274510 2024-12-09T08:17:54,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274510 2024-12-09T08:17:54,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274510) roll requested 2024-12-09T08:17:54,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274516 2024-12-09T08:17:54,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274510 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274516 2024-12-09T08:17:54,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274516) roll requested 2024-12-09T08:17:54,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274522 2024-12-09T08:17:54,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274516 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274522 2024-12-09T08:17:54,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274522) roll requested 2024-12-09T08:17:54,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274529 2024-12-09T08:17:54,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274529 2024-12-09T08:17:54,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274529) roll requested 2024-12-09T08:17:54,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274536 2024-12-09T08:17:54,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274529 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274536 2024-12-09T08:17:54,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274536) roll requested 2024-12-09T08:17:54,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274544 2024-12-09T08:17:54,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274544 2024-12-09T08:17:54,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274544) roll requested 2024-12-09T08:17:54,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274552 2024-12-09T08:17:54,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274552 2024-12-09T08:17:54,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274552) roll requested 2024-12-09T08:17:54,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274560 2024-12-09T08:17:54,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274560 2024-12-09T08:17:54,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274560) roll requested 2024-12-09T08:17:54,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274569 2024-12-09T08:17:54,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274569 2024-12-09T08:17:54,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274569) roll requested 2024-12-09T08:17:54,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274576 2024-12-09T08:17:54,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274576 2024-12-09T08:17:54,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274576) roll requested 2024-12-09T08:17:54,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274583 2024-12-09T08:17:54,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274583 2024-12-09T08:17:54,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274583) roll requested 2024-12-09T08:17:54,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274592 2024-12-09T08:17:54,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274583 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274592 2024-12-09T08:17:54,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,600 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274592) roll requested 2024-12-09T08:17:54,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274601 2024-12-09T08:17:54,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274601 2024-12-09T08:17:54,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274601) roll requested 2024-12-09T08:17:54,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274608 2024-12-09T08:17:54,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274608 2024-12-09T08:17:54,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,615 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274608) roll requested 2024-12-09T08:17:54,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274615 2024-12-09T08:17:54,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274615 2024-12-09T08:17:54,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,621 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274615) roll requested 2024-12-09T08:17:54,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274621 2024-12-09T08:17:54,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274615 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274621 2024-12-09T08:17:54,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274621) roll requested 2024-12-09T08:17:54,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274629 2024-12-09T08:17:54,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274629 2024-12-09T08:17:54,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274629) roll requested 2024-12-09T08:17:54,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274637 2024-12-09T08:17:54,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274637 2024-12-09T08:17:54,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,647 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274637) roll requested 2024-12-09T08:17:54,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274647 2024-12-09T08:17:54,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274637 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274647 2024-12-09T08:17:54,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274647) roll requested 2024-12-09T08:17:54,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274653 2024-12-09T08:17:54,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274653 2024-12-09T08:17:54,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,658 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274653) roll requested 2024-12-09T08:17:54,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274658 2024-12-09T08:17:54,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274658 2024-12-09T08:17:54,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274658) roll requested 2024-12-09T08:17:54,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274664 2024-12-09T08:17:54,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274658 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274664 2024-12-09T08:17:54,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274664) roll requested 2024-12-09T08:17:54,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274670 2024-12-09T08:17:54,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274670 2024-12-09T08:17:54,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274670) roll requested 2024-12-09T08:17:54,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274678 2024-12-09T08:17:54,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274670 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274678 2024-12-09T08:17:54,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274678) roll requested 2024-12-09T08:17:54,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274686 2024-12-09T08:17:54,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274686 2024-12-09T08:17:54,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274686) roll requested 2024-12-09T08:17:54,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274695 2024-12-09T08:17:54,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274686 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274695 2024-12-09T08:17:54,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,703 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274695) roll requested 2024-12-09T08:17:54,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274703 2024-12-09T08:17:54,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274703 2024-12-09T08:17:54,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274703) roll requested 2024-12-09T08:17:54,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274709 2024-12-09T08:17:54,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274703 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274709 2024-12-09T08:17:54,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,716 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274709) roll requested 2024-12-09T08:17:54,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274716 2024-12-09T08:17:54,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274716 2024-12-09T08:17:54,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,723 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274716) roll requested 2024-12-09T08:17:54,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274724 2024-12-09T08:17:54,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274716 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274724 2024-12-09T08:17:54,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,731 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274724) roll requested 2024-12-09T08:17:54,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274732 2024-12-09T08:17:54,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274724 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274732 2024-12-09T08:17:54,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,740 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274732) roll requested 2024-12-09T08:17:54,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274741 2024-12-09T08:17:54,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274741 2024-12-09T08:17:54,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274741) roll requested 2024-12-09T08:17:54,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274751 2024-12-09T08:17:54,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274741 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274751 2024-12-09T08:17:54,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274751) roll requested 2024-12-09T08:17:54,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274760 2024-12-09T08:17:54,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274760 2024-12-09T08:17:54,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,771 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274760) roll requested 2024-12-09T08:17:54,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274771 2024-12-09T08:17:54,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274771 2024-12-09T08:17:54,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274771) roll requested 2024-12-09T08:17:54,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274778 2024-12-09T08:17:54,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274771 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274778 2024-12-09T08:17:54,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274778) roll requested 2024-12-09T08:17:54,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274785 2024-12-09T08:17:54,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274785 2024-12-09T08:17:54,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,794 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274785) roll requested 2024-12-09T08:17:54,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274794 2024-12-09T08:17:54,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274794 2024-12-09T08:17:54,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274794) roll requested 2024-12-09T08:17:54,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274804 2024-12-09T08:17:54,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274794 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274804 2024-12-09T08:17:54,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,814 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274804) roll requested 2024-12-09T08:17:54,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274815 2024-12-09T08:17:54,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274815 2024-12-09T08:17:54,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274815) roll requested 2024-12-09T08:17:54,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274822 2024-12-09T08:17:54,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274815 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274822 2024-12-09T08:17:54,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274822) roll requested 2024-12-09T08:17:54,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274829 2024-12-09T08:17:54,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274829 2024-12-09T08:17:54,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274829) roll requested 2024-12-09T08:17:54,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274838 2024-12-09T08:17:54,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274838 2024-12-09T08:17:54,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,850 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274838) roll requested 2024-12-09T08:17:54,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274850 2024-12-09T08:17:54,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274838 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274850 2024-12-09T08:17:54,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274850) roll requested 2024-12-09T08:17:54,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274859 2024-12-09T08:17:54,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274859 2024-12-09T08:17:54,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274859) roll requested 2024-12-09T08:17:54,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274867 2024-12-09T08:17:54,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274859 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274867 2024-12-09T08:17:54,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274867) roll requested 2024-12-09T08:17:54,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274876 2024-12-09T08:17:54,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274876 2024-12-09T08:17:54,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274876) roll requested 2024-12-09T08:17:54,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274886 2024-12-09T08:17:54,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274886 2024-12-09T08:17:54,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274886) roll requested 2024-12-09T08:17:54,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274894 2024-12-09T08:17:54,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274886 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274894 2024-12-09T08:17:54,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274894) roll requested 2024-12-09T08:17:54,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274900 2024-12-09T08:17:54,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274900 2024-12-09T08:17:54,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274900) roll requested 2024-12-09T08:17:54,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274907 2024-12-09T08:17:54,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274907 2024-12-09T08:17:54,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274907) roll requested 2024-12-09T08:17:54,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274916 2024-12-09T08:17:54,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274907 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274916 2024-12-09T08:17:54,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274916) roll requested 2024-12-09T08:17:54,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274925 2024-12-09T08:17:54,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274925 2024-12-09T08:17:54,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274925) roll requested 2024-12-09T08:17:54,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274931 2024-12-09T08:17:54,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274931 2024-12-09T08:17:54,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274931) roll requested 2024-12-09T08:17:54,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274937 2024-12-09T08:17:54,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274937 2024-12-09T08:17:54,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274937) roll requested 2024-12-09T08:17:54,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274944 2024-12-09T08:17:54,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274944 2024-12-09T08:17:54,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274944) roll requested 2024-12-09T08:17:54,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274953 2024-12-09T08:17:54,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274953 2024-12-09T08:17:54,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274953) roll requested 2024-12-09T08:17:54,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274960 2024-12-09T08:17:54,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274960 2024-12-09T08:17:54,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,967 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274960) roll requested 2024-12-09T08:17:54,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274968 2024-12-09T08:17:54,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274968 2024-12-09T08:17:54,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,975 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274968) roll requested 2024-12-09T08:17:54,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274975 2024-12-09T08:17:54,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274975 2024-12-09T08:17:54,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,984 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274975) roll requested 2024-12-09T08:17:54,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274984 2024-12-09T08:17:54,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274975 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274984 2024-12-09T08:17:54,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:54,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:54,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274984) roll requested 2024-12-09T08:17:54,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732274992 2024-12-09T08:17:54,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:54,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274984 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274992 2024-12-09T08:17:55,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732274992) roll requested 2024-12-09T08:17:55,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275001 2024-12-09T08:17:55,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732274992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275001 2024-12-09T08:17:55,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275001) roll requested 2024-12-09T08:17:55,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275011 2024-12-09T08:17:55,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275001 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275011 2024-12-09T08:17:55,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,019 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275011) roll requested 2024-12-09T08:17:55,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275019 2024-12-09T08:17:55,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275019 2024-12-09T08:17:55,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,027 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275019) roll requested 2024-12-09T08:17:55,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275027 2024-12-09T08:17:55,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275019 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275027 2024-12-09T08:17:55,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275027) roll requested 2024-12-09T08:17:55,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275036 2024-12-09T08:17:55,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275027 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275036 2024-12-09T08:17:55,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,046 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275036) roll requested 2024-12-09T08:17:55,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275046 2024-12-09T08:17:55,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275036 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275046 2024-12-09T08:17:55,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,053 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275046) roll requested 2024-12-09T08:17:55,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275054 2024-12-09T08:17:55,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275046 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275054 2024-12-09T08:17:55,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,065 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275054) roll requested 2024-12-09T08:17:55,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275066 2024-12-09T08:17:55,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275066 2024-12-09T08:17:55,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,074 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275066) roll requested 2024-12-09T08:17:55,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275075 2024-12-09T08:17:55,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275066 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275075 2024-12-09T08:17:55,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275075) roll requested 2024-12-09T08:17:55,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275082 2024-12-09T08:17:55,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275075 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275082 2024-12-09T08:17:55,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,091 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275082) roll requested 2024-12-09T08:17:55,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275091 2024-12-09T08:17:55,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275091 2024-12-09T08:17:55,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,097 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275091) roll requested 2024-12-09T08:17:55,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275097 2024-12-09T08:17:55,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275091 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275097 2024-12-09T08:17:55,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,102 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275097) roll requested 2024-12-09T08:17:55,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275102 2024-12-09T08:17:55,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275102 2024-12-09T08:17:55,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275102) roll requested 2024-12-09T08:17:55,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275113 2024-12-09T08:17:55,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275102 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275113 2024-12-09T08:17:55,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,126 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275113) roll requested 2024-12-09T08:17:55,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275126 2024-12-09T08:17:55,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275126 2024-12-09T08:17:55,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,135 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275126) roll requested 2024-12-09T08:17:55,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275135 2024-12-09T08:17:55,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275126 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275135 2024-12-09T08:17:55,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275135) roll requested 2024-12-09T08:17:55,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275142 2024-12-09T08:17:55,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275142 2024-12-09T08:17:55,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,149 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275142) roll requested 2024-12-09T08:17:55,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275149 2024-12-09T08:17:55,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275149 2024-12-09T08:17:55,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,155 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275149) roll requested 2024-12-09T08:17:55,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275155 2024-12-09T08:17:55,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275149 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275155 2024-12-09T08:17:55,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,163 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275155) roll requested 2024-12-09T08:17:55,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275163 2024-12-09T08:17:55,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275155 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275163 2024-12-09T08:17:55,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275163) roll requested 2024-12-09T08:17:55,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275168 2024-12-09T08:17:55,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275163 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275168 2024-12-09T08:17:55,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275168) roll requested 2024-12-09T08:17:55,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275173 2024-12-09T08:17:55,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275173 2024-12-09T08:17:55,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275173) roll requested 2024-12-09T08:17:55,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275179 2024-12-09T08:17:55,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275179 2024-12-09T08:17:55,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,185 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275179) roll requested 2024-12-09T08:17:55,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275185 2024-12-09T08:17:55,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275185 2024-12-09T08:17:55,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,191 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275185) roll requested 2024-12-09T08:17:55,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275192 2024-12-09T08:17:55,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275192 2024-12-09T08:17:55,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,198 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275192) roll requested 2024-12-09T08:17:55,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275199 2024-12-09T08:17:55,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275192 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275199 2024-12-09T08:17:55,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,207 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275199) roll requested 2024-12-09T08:17:55,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275207 2024-12-09T08:17:55,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275199 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275207 2024-12-09T08:17:55,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275207) roll requested 2024-12-09T08:17:55,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275215 2024-12-09T08:17:55,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275207 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275215 2024-12-09T08:17:55,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275215) roll requested 2024-12-09T08:17:55,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275222 2024-12-09T08:17:55,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275222 2024-12-09T08:17:55,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275222) roll requested 2024-12-09T08:17:55,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275229 2024-12-09T08:17:55,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275229 2024-12-09T08:17:55,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275229) roll requested 2024-12-09T08:17:55,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275237 2024-12-09T08:17:55,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275237 2024-12-09T08:17:55,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275237) roll requested 2024-12-09T08:17:55,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275244 2024-12-09T08:17:55,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275244 2024-12-09T08:17:55,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275244) roll requested 2024-12-09T08:17:55,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275251 2024-12-09T08:17:55,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275251 2024-12-09T08:17:55,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275251) roll requested 2024-12-09T08:17:55,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275259 2024-12-09T08:17:55,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275259 2024-12-09T08:17:55,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275259) roll requested 2024-12-09T08:17:55,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275273 2024-12-09T08:17:55,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275273 2024-12-09T08:17:55,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275273) roll requested 2024-12-09T08:17:55,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275293 2024-12-09T08:17:55,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275293 2024-12-09T08:17:55,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,309 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275293) roll requested 2024-12-09T08:17:55,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275309 2024-12-09T08:17:55,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275309 2024-12-09T08:17:55,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275309) roll requested 2024-12-09T08:17:55,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275315 2024-12-09T08:17:55,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275309 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275315 2024-12-09T08:17:55,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275315) roll requested 2024-12-09T08:17:55,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275320 2024-12-09T08:17:55,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275315 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275320 2024-12-09T08:17:55,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,326 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275320) roll requested 2024-12-09T08:17:55,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275326 2024-12-09T08:17:55,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275326 2024-12-09T08:17:55,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275326) roll requested 2024-12-09T08:17:55,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275333 2024-12-09T08:17:55,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275326 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275333 2024-12-09T08:17:55,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275333) roll requested 2024-12-09T08:17:55,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275339 2024-12-09T08:17:55,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275339 2024-12-09T08:17:55,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,346 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275339) roll requested 2024-12-09T08:17:55,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275347 2024-12-09T08:17:55,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275347 2024-12-09T08:17:55,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275347) roll requested 2024-12-09T08:17:55,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275354 2024-12-09T08:17:55,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275354 2024-12-09T08:17:55,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275354) roll requested 2024-12-09T08:17:55,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275364 2024-12-09T08:17:55,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275354 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275364 2024-12-09T08:17:55,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,373 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275364) roll requested 2024-12-09T08:17:55,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275373 2024-12-09T08:17:55,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275364 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275373 2024-12-09T08:17:55,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275373) roll requested 2024-12-09T08:17:55,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275387 2024-12-09T08:17:55,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275373 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275387 2024-12-09T08:17:55,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,395 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275387) roll requested 2024-12-09T08:17:55,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275395 2024-12-09T08:17:55,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275395 2024-12-09T08:17:55,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275395) roll requested 2024-12-09T08:17:55,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275401 2024-12-09T08:17:55,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275395 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275401 2024-12-09T08:17:55,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275401) roll requested 2024-12-09T08:17:55,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275406 2024-12-09T08:17:55,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275406 2024-12-09T08:17:55,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275406) roll requested 2024-12-09T08:17:55,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275413 2024-12-09T08:17:55,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275413 2024-12-09T08:17:55,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,418 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275413) roll requested 2024-12-09T08:17:55,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275419 2024-12-09T08:17:55,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275419 2024-12-09T08:17:55,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,423 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275419) roll requested 2024-12-09T08:17:55,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275423 2024-12-09T08:17:55,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275419 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275423 2024-12-09T08:17:55,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275423) roll requested 2024-12-09T08:17:55,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275428 2024-12-09T08:17:55,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275423 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275428 2024-12-09T08:17:55,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,433 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275428) roll requested 2024-12-09T08:17:55,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275433 2024-12-09T08:17:55,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275433 2024-12-09T08:17:55,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275433) roll requested 2024-12-09T08:17:55,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275440 2024-12-09T08:17:55,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275440 2024-12-09T08:17:55,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275440) roll requested 2024-12-09T08:17:55,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275446 2024-12-09T08:17:55,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275446 2024-12-09T08:17:55,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275446) roll requested 2024-12-09T08:17:55,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275453 2024-12-09T08:17:55,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275446 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275453 2024-12-09T08:17:55,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275453) roll requested 2024-12-09T08:17:55,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275461 2024-12-09T08:17:55,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275461 2024-12-09T08:17:55,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,469 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275461) roll requested 2024-12-09T08:17:55,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275469 2024-12-09T08:17:55,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275469 2024-12-09T08:17:55,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275469) roll requested 2024-12-09T08:17:55,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275475 2024-12-09T08:17:55,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275469 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275475 2024-12-09T08:17:55,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275475) roll requested 2024-12-09T08:17:55,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275483 2024-12-09T08:17:55,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275475 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275483 2024-12-09T08:17:55,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,491 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275483) roll requested 2024-12-09T08:17:55,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275491 2024-12-09T08:17:55,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275491 2024-12-09T08:17:55,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,499 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275491) roll requested 2024-12-09T08:17:55,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275499 2024-12-09T08:17:55,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275499 2024-12-09T08:17:55,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275499) roll requested 2024-12-09T08:17:55,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275507 2024-12-09T08:17:55,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275499 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275507 2024-12-09T08:17:55,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,517 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275507) roll requested 2024-12-09T08:17:55,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275517 2024-12-09T08:17:55,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275517 2024-12-09T08:17:55,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,525 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275517) roll requested 2024-12-09T08:17:55,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275525 2024-12-09T08:17:55,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275525 2024-12-09T08:17:55,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275525) roll requested 2024-12-09T08:17:55,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275531 2024-12-09T08:17:55,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275525 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275531 2024-12-09T08:17:55,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275531) roll requested 2024-12-09T08:17:55,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275540 2024-12-09T08:17:55,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275540 2024-12-09T08:17:55,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275540) roll requested 2024-12-09T08:17:55,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275548 2024-12-09T08:17:55,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275540 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275548 2024-12-09T08:17:55,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275548) roll requested 2024-12-09T08:17:55,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275557 2024-12-09T08:17:55,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275557 2024-12-09T08:17:55,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275557) roll requested 2024-12-09T08:17:55,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275566 2024-12-09T08:17:55,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275566 2024-12-09T08:17:55,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,573 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275566) roll requested 2024-12-09T08:17:55,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275573 2024-12-09T08:17:55,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275566 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275573 2024-12-09T08:17:55,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,581 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275573) roll requested 2024-12-09T08:17:55,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275581 2024-12-09T08:17:55,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275573 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275581 2024-12-09T08:17:55,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,590 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275581) roll requested 2024-12-09T08:17:55,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275590 2024-12-09T08:17:55,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275590 2024-12-09T08:17:55,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275590) roll requested 2024-12-09T08:17:55,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275599 2024-12-09T08:17:55,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275590 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275599 2024-12-09T08:17:55,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275599) roll requested 2024-12-09T08:17:55,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275609 2024-12-09T08:17:55,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275609 2024-12-09T08:17:55,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275609) roll requested 2024-12-09T08:17:55,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275617 2024-12-09T08:17:55,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275617 2024-12-09T08:17:55,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275617) roll requested 2024-12-09T08:17:55,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275624 2024-12-09T08:17:55,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275624 2024-12-09T08:17:55,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275624) roll requested 2024-12-09T08:17:55,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275631 2024-12-09T08:17:55,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275624 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275631 2024-12-09T08:17:55,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275631) roll requested 2024-12-09T08:17:55,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275639 2024-12-09T08:17:55,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275639 2024-12-09T08:17:55,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275639) roll requested 2024-12-09T08:17:55,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275648 2024-12-09T08:17:55,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275648 2024-12-09T08:17:55,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275648) roll requested 2024-12-09T08:17:55,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275656 2024-12-09T08:17:55,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275656 2024-12-09T08:17:55,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,661 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275656) roll requested 2024-12-09T08:17:55,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275661 2024-12-09T08:17:55,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275661 2024-12-09T08:17:55,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275661) roll requested 2024-12-09T08:17:55,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275667 2024-12-09T08:17:55,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275667 2024-12-09T08:17:55,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275667) roll requested 2024-12-09T08:17:55,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275677 2024-12-09T08:17:55,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275677 2024-12-09T08:17:55,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275677) roll requested 2024-12-09T08:17:55,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275683 2024-12-09T08:17:55,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275683 2024-12-09T08:17:55,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275683) roll requested 2024-12-09T08:17:55,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275690 2024-12-09T08:17:55,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275690 2024-12-09T08:17:55,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275690) roll requested 2024-12-09T08:17:55,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275700 2024-12-09T08:17:55,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275700 2024-12-09T08:17:55,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275700) roll requested 2024-12-09T08:17:55,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275709 2024-12-09T08:17:55,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275709 2024-12-09T08:17:55,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,716 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275709) roll requested 2024-12-09T08:17:55,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275717 2024-12-09T08:17:55,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275717 2024-12-09T08:17:55,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,722 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275717) roll requested 2024-12-09T08:17:55,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275722 2024-12-09T08:17:55,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275717 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275722 2024-12-09T08:17:55,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275722) roll requested 2024-12-09T08:17:55,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275730 2024-12-09T08:17:55,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275722 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275730 2024-12-09T08:17:55,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275730) roll requested 2024-12-09T08:17:55,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275736 2024-12-09T08:17:55,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275730 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275736 2024-12-09T08:17:55,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,740 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275736) roll requested 2024-12-09T08:17:55,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275741 2024-12-09T08:17:55,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275741 2024-12-09T08:17:55,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275741) roll requested 2024-12-09T08:17:55,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275751 2024-12-09T08:17:55,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275741 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275751 2024-12-09T08:17:55,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275751) roll requested 2024-12-09T08:17:55,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275757 2024-12-09T08:17:55,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275757 2024-12-09T08:17:55,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,762 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275757) roll requested 2024-12-09T08:17:55,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275762 2024-12-09T08:17:55,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275762 2024-12-09T08:17:55,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,767 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275762) roll requested 2024-12-09T08:17:55,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275767 2024-12-09T08:17:55,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275762 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275767 2024-12-09T08:17:55,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275767) roll requested 2024-12-09T08:17:55,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275772 2024-12-09T08:17:55,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275772 2024-12-09T08:17:55,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275772) roll requested 2024-12-09T08:17:55,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275778 2024-12-09T08:17:55,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275778 2024-12-09T08:17:55,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275778) roll requested 2024-12-09T08:17:55,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275783 2024-12-09T08:17:55,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275783 2024-12-09T08:17:55,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275783) roll requested 2024-12-09T08:17:55,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275788 2024-12-09T08:17:55,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275788 2024-12-09T08:17:55,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275788) roll requested 2024-12-09T08:17:55,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275795 2024-12-09T08:17:55,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275795 2024-12-09T08:17:55,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275795) roll requested 2024-12-09T08:17:55,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275800 2024-12-09T08:17:55,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275800 2024-12-09T08:17:55,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275800) roll requested 2024-12-09T08:17:55,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275806 2024-12-09T08:17:55,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275806 2024-12-09T08:17:55,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,811 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275806) roll requested 2024-12-09T08:17:55,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275811 2024-12-09T08:17:55,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275811 2024-12-09T08:17:55,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275811) roll requested 2024-12-09T08:17:55,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275817 2024-12-09T08:17:55,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275811 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275817 2024-12-09T08:17:55,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275817) roll requested 2024-12-09T08:17:55,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275822 2024-12-09T08:17:55,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275822 2024-12-09T08:17:55,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275822) roll requested 2024-12-09T08:17:55,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275827 2024-12-09T08:17:55,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275827 2024-12-09T08:17:55,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275827) roll requested 2024-12-09T08:17:55,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275832 2024-12-09T08:17:55,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275832 2024-12-09T08:17:55,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275832) roll requested 2024-12-09T08:17:55,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275838 2024-12-09T08:17:55,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275838 2024-12-09T08:17:55,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275838) roll requested 2024-12-09T08:17:55,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275843 2024-12-09T08:17:55,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275838 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275843 2024-12-09T08:17:55,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275843) roll requested 2024-12-09T08:17:55,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275848 2024-12-09T08:17:55,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275843 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275848 2024-12-09T08:17:55,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275848) roll requested 2024-12-09T08:17:55,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275853 2024-12-09T08:17:55,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275853 2024-12-09T08:17:55,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275853) roll requested 2024-12-09T08:17:55,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275859 2024-12-09T08:17:55,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275859 2024-12-09T08:17:55,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275859) roll requested 2024-12-09T08:17:55,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275864 2024-12-09T08:17:55,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275859 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275864 2024-12-09T08:17:55,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,870 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275864) roll requested 2024-12-09T08:17:55,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275870 2024-12-09T08:17:55,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275864 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275870 2024-12-09T08:17:55,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275870) roll requested 2024-12-09T08:17:55,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275875 2024-12-09T08:17:55,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275870 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275875 2024-12-09T08:17:55,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275875) roll requested 2024-12-09T08:17:55,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275881 2024-12-09T08:17:55,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275875 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275881 2024-12-09T08:17:55,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,886 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275881) roll requested 2024-12-09T08:17:55,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275886 2024-12-09T08:17:55,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275886 2024-12-09T08:17:55,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,891 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275886) roll requested 2024-12-09T08:17:55,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275891 2024-12-09T08:17:55,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275886 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275891 2024-12-09T08:17:55,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275891) roll requested 2024-12-09T08:17:55,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275896 2024-12-09T08:17:55,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275896 2024-12-09T08:17:55,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275896) roll requested 2024-12-09T08:17:55,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275902 2024-12-09T08:17:55,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275902 2024-12-09T08:17:55,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275902) roll requested 2024-12-09T08:17:55,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275908 2024-12-09T08:17:55,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275902 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275908 2024-12-09T08:17:55,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275908) roll requested 2024-12-09T08:17:55,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275913 2024-12-09T08:17:55,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275913 2024-12-09T08:17:55,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,919 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275913) roll requested 2024-12-09T08:17:55,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275919 2024-12-09T08:17:55,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275913 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275919 2024-12-09T08:17:55,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275919) roll requested 2024-12-09T08:17:55,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275925 2024-12-09T08:17:55,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275925 2024-12-09T08:17:55,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,930 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275925) roll requested 2024-12-09T08:17:55,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275930 2024-12-09T08:17:55,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275930 2024-12-09T08:17:55,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275930) roll requested 2024-12-09T08:17:55,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275936 2024-12-09T08:17:55,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275930 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275936 2024-12-09T08:17:55,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275936) roll requested 2024-12-09T08:17:55,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275941 2024-12-09T08:17:55,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275941 2024-12-09T08:17:55,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275941) roll requested 2024-12-09T08:17:55,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275946 2024-12-09T08:17:55,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275941 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275946 2024-12-09T08:17:55,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275946) roll requested 2024-12-09T08:17:55,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275952 2024-12-09T08:17:55,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275946 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275952 2024-12-09T08:17:55,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275952) roll requested 2024-12-09T08:17:55,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275958 2024-12-09T08:17:55,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275958 2024-12-09T08:17:55,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275958) roll requested 2024-12-09T08:17:55,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275965 2024-12-09T08:17:55,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275965 2024-12-09T08:17:55,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,970 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275965) roll requested 2024-12-09T08:17:55,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275971 2024-12-09T08:17:55,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275971 2024-12-09T08:17:55,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,975 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275971) roll requested 2024-12-09T08:17:55,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275976 2024-12-09T08:17:55,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275971 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275976 2024-12-09T08:17:55,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275976) roll requested 2024-12-09T08:17:55,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275982 2024-12-09T08:17:55,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275982 2024-12-09T08:17:55,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275982) roll requested 2024-12-09T08:17:55,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275988 2024-12-09T08:17:55,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:55,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275988 2024-12-09T08:17:55,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:55,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:55,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275988) roll requested 2024-12-09T08:17:55,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732275996 2024-12-09T08:17:56,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275996 2024-12-09T08:17:56,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732275996) roll requested 2024-12-09T08:17:56,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276005 2024-12-09T08:17:56,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732275996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276005 2024-12-09T08:17:56,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276005) roll requested 2024-12-09T08:17:56,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276011 2024-12-09T08:17:56,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276005 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276011 2024-12-09T08:17:56,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276011) roll requested 2024-12-09T08:17:56,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276017 2024-12-09T08:17:56,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276017 2024-12-09T08:17:56,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,024 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276017) roll requested 2024-12-09T08:17:56,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276024 2024-12-09T08:17:56,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276024 2024-12-09T08:17:56,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,031 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276024) roll requested 2024-12-09T08:17:56,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276032 2024-12-09T08:17:56,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276024 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276032 2024-12-09T08:17:56,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276032) roll requested 2024-12-09T08:17:56,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276040 2024-12-09T08:17:56,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276040 2024-12-09T08:17:56,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276040) roll requested 2024-12-09T08:17:56,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276048 2024-12-09T08:17:56,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276048 2024-12-09T08:17:56,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276048) roll requested 2024-12-09T08:17:56,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276059 2024-12-09T08:17:56,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276059 2024-12-09T08:17:56,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,066 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276059) roll requested 2024-12-09T08:17:56,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276067 2024-12-09T08:17:56,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276059 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276067 2024-12-09T08:17:56,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276067) roll requested 2024-12-09T08:17:56,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276073 2024-12-09T08:17:56,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276067 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276073 2024-12-09T08:17:56,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,081 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276073) roll requested 2024-12-09T08:17:56,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276081 2024-12-09T08:17:56,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276081 2024-12-09T08:17:56,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276081) roll requested 2024-12-09T08:17:56,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276089 2024-12-09T08:17:56,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276081 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276089 2024-12-09T08:17:56,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276089) roll requested 2024-12-09T08:17:56,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276096 2024-12-09T08:17:56,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276096 2024-12-09T08:17:56,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276096) roll requested 2024-12-09T08:17:56,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276105 2024-12-09T08:17:56,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276105 2024-12-09T08:17:56,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,114 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276105) roll requested 2024-12-09T08:17:56,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276115 2024-12-09T08:17:56,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276115 2024-12-09T08:17:56,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276115) roll requested 2024-12-09T08:17:56,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276122 2024-12-09T08:17:56,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276115 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276122 2024-12-09T08:17:56,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,133 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276122) roll requested 2024-12-09T08:17:56,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276133 2024-12-09T08:17:56,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276122 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276133 2024-12-09T08:17:56,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,139 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276133) roll requested 2024-12-09T08:17:56,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276139 2024-12-09T08:17:56,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276139 2024-12-09T08:17:56,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,144 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276139) roll requested 2024-12-09T08:17:56,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276144 2024-12-09T08:17:56,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276139 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276144 2024-12-09T08:17:56,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,149 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276144) roll requested 2024-12-09T08:17:56,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276149 2024-12-09T08:17:56,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276144 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276149 2024-12-09T08:17:56,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276149) roll requested 2024-12-09T08:17:56,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276156 2024-12-09T08:17:56,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276149 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276156 2024-12-09T08:17:56,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,162 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276156) roll requested 2024-12-09T08:17:56,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276163 2024-12-09T08:17:56,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276156 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276163 2024-12-09T08:17:56,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276163) roll requested 2024-12-09T08:17:56,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276170 2024-12-09T08:17:56,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276163 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276170 2024-12-09T08:17:56,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,178 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276170) roll requested 2024-12-09T08:17:56,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276178 2024-12-09T08:17:56,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276170 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276178 2024-12-09T08:17:56,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,187 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276178) roll requested 2024-12-09T08:17:56,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276188 2024-12-09T08:17:56,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276178 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276188 2024-12-09T08:17:56,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276188) roll requested 2024-12-09T08:17:56,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276195 2024-12-09T08:17:56,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276195 2024-12-09T08:17:56,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276195) roll requested 2024-12-09T08:17:56,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276202 2024-12-09T08:17:56,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276202 2024-12-09T08:17:56,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276202) roll requested 2024-12-09T08:17:56,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276208 2024-12-09T08:17:56,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276208 2024-12-09T08:17:56,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,215 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276208) roll requested 2024-12-09T08:17:56,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276215 2024-12-09T08:17:56,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276215 2024-12-09T08:17:56,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,221 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276215) roll requested 2024-12-09T08:17:56,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276221 2024-12-09T08:17:56,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276221 2024-12-09T08:17:56,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276221) roll requested 2024-12-09T08:17:56,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276228 2024-12-09T08:17:56,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276221 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276228 2024-12-09T08:17:56,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,235 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276228) roll requested 2024-12-09T08:17:56,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276235 2024-12-09T08:17:56,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276235 2024-12-09T08:17:56,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,243 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276235) roll requested 2024-12-09T08:17:56,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276243 2024-12-09T08:17:56,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276235 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276243 2024-12-09T08:17:56,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276243) roll requested 2024-12-09T08:17:56,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276253 2024-12-09T08:17:56,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276243 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276253 2024-12-09T08:17:56,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,258 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276253) roll requested 2024-12-09T08:17:56,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276259 2024-12-09T08:17:56,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276259 2024-12-09T08:17:56,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,265 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276259) roll requested 2024-12-09T08:17:56,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276265 2024-12-09T08:17:56,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276265 2024-12-09T08:17:56,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276265) roll requested 2024-12-09T08:17:56,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276273 2024-12-09T08:17:56,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276273 2024-12-09T08:17:56,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276273) roll requested 2024-12-09T08:17:56,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276282 2024-12-09T08:17:56,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276282 2024-12-09T08:17:56,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276282) roll requested 2024-12-09T08:17:56,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276289 2024-12-09T08:17:56,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276282 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276289 2024-12-09T08:17:56,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,297 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276289) roll requested 2024-12-09T08:17:56,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276298 2024-12-09T08:17:56,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276298 2024-12-09T08:17:56,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276298) roll requested 2024-12-09T08:17:56,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276309 2024-12-09T08:17:56,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276309 2024-12-09T08:17:56,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,316 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276309) roll requested 2024-12-09T08:17:56,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276316 2024-12-09T08:17:56,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276309 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276316 2024-12-09T08:17:56,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,323 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276316) roll requested 2024-12-09T08:17:56,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276323 2024-12-09T08:17:56,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276316 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276323 2024-12-09T08:17:56,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276323) roll requested 2024-12-09T08:17:56,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276331 2024-12-09T08:17:56,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276331 2024-12-09T08:17:56,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276331) roll requested 2024-12-09T08:17:56,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276340 2024-12-09T08:17:56,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276340 2024-12-09T08:17:56,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276340) roll requested 2024-12-09T08:17:56,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276349 2024-12-09T08:17:56,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276349 2024-12-09T08:17:56,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,358 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276349) roll requested 2024-12-09T08:17:56,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276359 2024-12-09T08:17:56,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276349 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276359 2024-12-09T08:17:56,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276359) roll requested 2024-12-09T08:17:56,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276369 2024-12-09T08:17:56,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276369 2024-12-09T08:17:56,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276369) roll requested 2024-12-09T08:17:56,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276376 2024-12-09T08:17:56,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276376 2024-12-09T08:17:56,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276376) roll requested 2024-12-09T08:17:56,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276382 2024-12-09T08:17:56,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276382 2024-12-09T08:17:56,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,390 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276382) roll requested 2024-12-09T08:17:56,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276391 2024-12-09T08:17:56,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276391 2024-12-09T08:17:56,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,398 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276391) roll requested 2024-12-09T08:17:56,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276398 2024-12-09T08:17:56,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276391 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276398 2024-12-09T08:17:56,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276398) roll requested 2024-12-09T08:17:56,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276405 2024-12-09T08:17:56,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276398 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276405 2024-12-09T08:17:56,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276405) roll requested 2024-12-09T08:17:56,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276416 2024-12-09T08:17:56,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,424 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,424 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276405 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276416 2024-12-09T08:17:56,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,425 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276416) roll requested 2024-12-09T08:17:56,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276425 2024-12-09T08:17:56,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276416 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276425 2024-12-09T08:17:56,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,435 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276425) roll requested 2024-12-09T08:17:56,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276435 2024-12-09T08:17:56,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276435 2024-12-09T08:17:56,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276435) roll requested 2024-12-09T08:17:56,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276467 2024-12-09T08:17:56,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276435 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276467 2024-12-09T08:17:56,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276467) roll requested 2024-12-09T08:17:56,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276474 2024-12-09T08:17:56,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276474 2024-12-09T08:17:56,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276474) roll requested 2024-12-09T08:17:56,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276482 2024-12-09T08:17:56,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276482 2024-12-09T08:17:56,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276482) roll requested 2024-12-09T08:17:56,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276489 2024-12-09T08:17:56,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276482 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276489 2024-12-09T08:17:56,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276489) roll requested 2024-12-09T08:17:56,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276496 2024-12-09T08:17:56,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276496 2024-12-09T08:17:56,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,503 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276496) roll requested 2024-12-09T08:17:56,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276503 2024-12-09T08:17:56,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276503 2024-12-09T08:17:56,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,511 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276503) roll requested 2024-12-09T08:17:56,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276511 2024-12-09T08:17:56,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276511 2024-12-09T08:17:56,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,518 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276511) roll requested 2024-12-09T08:17:56,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276518 2024-12-09T08:17:56,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276511 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276518 2024-12-09T08:17:56,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276518) roll requested 2024-12-09T08:17:56,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276527 2024-12-09T08:17:56,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276518 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276527 2024-12-09T08:17:56,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276527) roll requested 2024-12-09T08:17:56,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276536 2024-12-09T08:17:56,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276536 2024-12-09T08:17:56,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276536) roll requested 2024-12-09T08:17:56,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276546 2024-12-09T08:17:56,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276546 2024-12-09T08:17:56,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276546) roll requested 2024-12-09T08:17:56,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276557 2024-12-09T08:17:56,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276546 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276557 2024-12-09T08:17:56,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,563 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276557) roll requested 2024-12-09T08:17:56,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276563 2024-12-09T08:17:56,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276563 2024-12-09T08:17:56,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276563) roll requested 2024-12-09T08:17:56,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276569 2024-12-09T08:17:56,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276563 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276569 2024-12-09T08:17:56,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276569) roll requested 2024-12-09T08:17:56,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276577 2024-12-09T08:17:56,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276577 2024-12-09T08:17:56,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276577) roll requested 2024-12-09T08:17:56,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276584 2024-12-09T08:17:56,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276584 2024-12-09T08:17:56,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276584) roll requested 2024-12-09T08:17:56,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276593 2024-12-09T08:17:56,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276593 2024-12-09T08:17:56,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,602 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276593) roll requested 2024-12-09T08:17:56,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276602 2024-12-09T08:17:56,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276602 2024-12-09T08:17:56,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276602) roll requested 2024-12-09T08:17:56,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276612 2024-12-09T08:17:56,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276612 2024-12-09T08:17:56,622 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,622 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,622 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276612) roll requested 2024-12-09T08:17:56,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276622 2024-12-09T08:17:56,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276622 2024-12-09T08:17:56,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276622) roll requested 2024-12-09T08:17:56,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276628 2024-12-09T08:17:56,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276622 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276628 2024-12-09T08:17:56,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,633 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276628) roll requested 2024-12-09T08:17:56,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276634 2024-12-09T08:17:56,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276634 2024-12-09T08:17:56,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,641 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276634) roll requested 2024-12-09T08:17:56,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276642 2024-12-09T08:17:56,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276634 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276642 2024-12-09T08:17:56,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276642) roll requested 2024-12-09T08:17:56,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276650 2024-12-09T08:17:56,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276642 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276650 2024-12-09T08:17:56,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276650) roll requested 2024-12-09T08:17:56,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276659 2024-12-09T08:17:56,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276650 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276659 2024-12-09T08:17:56,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276659) roll requested 2024-12-09T08:17:56,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276669 2024-12-09T08:17:56,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276659 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276669 2024-12-09T08:17:56,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,678 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276669) roll requested 2024-12-09T08:17:56,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276678 2024-12-09T08:17:56,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276678 2024-12-09T08:17:56,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276678) roll requested 2024-12-09T08:17:56,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276685 2024-12-09T08:17:56,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276685 2024-12-09T08:17:56,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276685) roll requested 2024-12-09T08:17:56,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276693 2024-12-09T08:17:56,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276693 2024-12-09T08:17:56,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,699 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276693) roll requested 2024-12-09T08:17:56,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276700 2024-12-09T08:17:56,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276700 2024-12-09T08:17:56,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276700) roll requested 2024-12-09T08:17:56,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276707 2024-12-09T08:17:56,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276707 2024-12-09T08:17:56,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,715 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276707) roll requested 2024-12-09T08:17:56,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276716 2024-12-09T08:17:56,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276707 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276716 2024-12-09T08:17:56,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276716) roll requested 2024-12-09T08:17:56,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276725 2024-12-09T08:17:56,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276716 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276725 2024-12-09T08:17:56,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276725) roll requested 2024-12-09T08:17:56,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276732 2024-12-09T08:17:56,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276732 2024-12-09T08:17:56,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,740 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276732) roll requested 2024-12-09T08:17:56,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276740 2024-12-09T08:17:56,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276740 2024-12-09T08:17:56,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,748 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276740) roll requested 2024-12-09T08:17:56,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276749 2024-12-09T08:17:56,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276740 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276749 2024-12-09T08:17:56,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,755 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276749) roll requested 2024-12-09T08:17:56,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276755 2024-12-09T08:17:56,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276755 2024-12-09T08:17:56,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276755) roll requested 2024-12-09T08:17:56,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276761 2024-12-09T08:17:56,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276755 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276761 2024-12-09T08:17:56,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,767 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276761) roll requested 2024-12-09T08:17:56,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276768 2024-12-09T08:17:56,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276768 2024-12-09T08:17:56,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276768) roll requested 2024-12-09T08:17:56,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276776 2024-12-09T08:17:56,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276776 2024-12-09T08:17:56,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276776) roll requested 2024-12-09T08:17:56,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276784 2024-12-09T08:17:56,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276784 2024-12-09T08:17:56,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276784) roll requested 2024-12-09T08:17:56,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276792 2024-12-09T08:17:56,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276792 2024-12-09T08:17:56,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,798 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276792) roll requested 2024-12-09T08:17:56,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276798 2024-12-09T08:17:56,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276792 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276798 2024-12-09T08:17:56,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,810 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276798) roll requested 2024-12-09T08:17:56,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276811 2024-12-09T08:17:56,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276798 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276811 2024-12-09T08:17:56,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,817 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276811) roll requested 2024-12-09T08:17:56,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276818 2024-12-09T08:17:56,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276811 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276818 2024-12-09T08:17:56,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,824 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276818) roll requested 2024-12-09T08:17:56,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276824 2024-12-09T08:17:56,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276824 2024-12-09T08:17:56,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276824) roll requested 2024-12-09T08:17:56,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276832 2024-12-09T08:17:56,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276824 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276832 2024-12-09T08:17:56,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,838 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276832) roll requested 2024-12-09T08:17:56,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276838 2024-12-09T08:17:56,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276838 2024-12-09T08:17:56,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276838) roll requested 2024-12-09T08:17:56,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276843 2024-12-09T08:17:56,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276838 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276843 2024-12-09T08:17:56,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276843) roll requested 2024-12-09T08:17:56,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276848 2024-12-09T08:17:56,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276843 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276848 2024-12-09T08:17:56,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276848) roll requested 2024-12-09T08:17:56,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276855 2024-12-09T08:17:56,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276855 2024-12-09T08:17:56,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276855) roll requested 2024-12-09T08:17:56,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276863 2024-12-09T08:17:56,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276855 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276863 2024-12-09T08:17:56,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,870 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276863) roll requested 2024-12-09T08:17:56,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276870 2024-12-09T08:17:56,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276863 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276870 2024-12-09T08:17:56,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276870) roll requested 2024-12-09T08:17:56,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276877 2024-12-09T08:17:56,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276870 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276877 2024-12-09T08:17:56,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,886 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276877) roll requested 2024-12-09T08:17:56,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276886 2024-12-09T08:17:56,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276877 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276886 2024-12-09T08:17:56,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,897 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276886) roll requested 2024-12-09T08:17:56,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276897 2024-12-09T08:17:56,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276886 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276897 2024-12-09T08:17:56,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,905 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276897) roll requested 2024-12-09T08:17:56,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276905 2024-12-09T08:17:56,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276897 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276905 2024-12-09T08:17:56,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,909 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276905) roll requested 2024-12-09T08:17:56,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276910 2024-12-09T08:17:56,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276910 2024-12-09T08:17:56,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,915 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276910) roll requested 2024-12-09T08:17:56,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276915 2024-12-09T08:17:56,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276910 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276915 2024-12-09T08:17:56,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276915) roll requested 2024-12-09T08:17:56,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276921 2024-12-09T08:17:56,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276921 2024-12-09T08:17:56,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276921) roll requested 2024-12-09T08:17:56,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276928 2024-12-09T08:17:56,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276928 2024-12-09T08:17:56,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276928) roll requested 2024-12-09T08:17:56,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276935 2024-12-09T08:17:56,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276935 2024-12-09T08:17:56,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,943 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276935) roll requested 2024-12-09T08:17:56,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276943 2024-12-09T08:17:56,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276935 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276943 2024-12-09T08:17:56,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276943) roll requested 2024-12-09T08:17:56,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276952 2024-12-09T08:17:56,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:56,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276943 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276952 2024-12-09T08:17:56,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:56,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276943 is not closed yet, will try archiving it next time 2024-12-09T08:17:56,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:56,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276952) roll requested 2024-12-09T08:17:56,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732276960 2024-12-09T08:17:57,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276960 2024-12-09T08:17:57,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732276960) roll requested 2024-12-09T08:17:57,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277001 2024-12-09T08:17:57,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732276960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277001 2024-12-09T08:17:57,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277001) roll requested 2024-12-09T08:17:57,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277008 2024-12-09T08:17:57,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277001 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277008 2024-12-09T08:17:57,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277008) roll requested 2024-12-09T08:17:57,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277016 2024-12-09T08:17:57,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277016 2024-12-09T08:17:57,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277016) roll requested 2024-12-09T08:17:57,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277025 2024-12-09T08:17:57,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277025 2024-12-09T08:17:57,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277025) roll requested 2024-12-09T08:17:57,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277036 2024-12-09T08:17:57,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277036 2024-12-09T08:17:57,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,044 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277036) roll requested 2024-12-09T08:17:57,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277045 2024-12-09T08:17:57,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277036 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277045 2024-12-09T08:17:57,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,053 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277045) roll requested 2024-12-09T08:17:57,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277054 2024-12-09T08:17:57,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277045 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277054 2024-12-09T08:17:57,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277054) roll requested 2024-12-09T08:17:57,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277061 2024-12-09T08:17:57,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277061 2024-12-09T08:17:57,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277061) roll requested 2024-12-09T08:17:57,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277069 2024-12-09T08:17:57,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277061 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277069 2024-12-09T08:17:57,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277069) roll requested 2024-12-09T08:17:57,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277076 2024-12-09T08:17:57,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277076 2024-12-09T08:17:57,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277076) roll requested 2024-12-09T08:17:57,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277084 2024-12-09T08:17:57,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277084 2024-12-09T08:17:57,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277084) roll requested 2024-12-09T08:17:57,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277093 2024-12-09T08:17:57,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277084 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277093 2024-12-09T08:17:57,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,101 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277093) roll requested 2024-12-09T08:17:57,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277102 2024-12-09T08:17:57,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277102 2024-12-09T08:17:57,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277102) roll requested 2024-12-09T08:17:57,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277112 2024-12-09T08:17:57,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277102 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277112 2024-12-09T08:17:57,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,118 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277112) roll requested 2024-12-09T08:17:57,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277118 2024-12-09T08:17:57,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277118 2024-12-09T08:17:57,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277118) roll requested 2024-12-09T08:17:57,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277124 2024-12-09T08:17:57,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277118 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277124 2024-12-09T08:17:57,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277124) roll requested 2024-12-09T08:17:57,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277130 2024-12-09T08:17:57,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277130 2024-12-09T08:17:57,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277130) roll requested 2024-12-09T08:17:57,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277137 2024-12-09T08:17:57,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277130 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277137 2024-12-09T08:17:57,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277137) roll requested 2024-12-09T08:17:57,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277146 2024-12-09T08:17:57,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277146 2024-12-09T08:17:57,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277146) roll requested 2024-12-09T08:17:57,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277151 2024-12-09T08:17:57,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277146 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277151 2024-12-09T08:17:57,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277151) roll requested 2024-12-09T08:17:57,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277157 2024-12-09T08:17:57,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277157 2024-12-09T08:17:57,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277157) roll requested 2024-12-09T08:17:57,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277170 2024-12-09T08:17:57,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277170 2024-12-09T08:17:57,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,175 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277170) roll requested 2024-12-09T08:17:57,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277175 2024-12-09T08:17:57,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277170 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277175 2024-12-09T08:17:57,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277175) roll requested 2024-12-09T08:17:57,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277180 2024-12-09T08:17:57,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277175 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277180 2024-12-09T08:17:57,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,185 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277180) roll requested 2024-12-09T08:17:57,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277186 2024-12-09T08:17:57,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277186 2024-12-09T08:17:57,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277186) roll requested 2024-12-09T08:17:57,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277192 2024-12-09T08:17:57,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277186 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277192 2024-12-09T08:17:57,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,198 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277192) roll requested 2024-12-09T08:17:57,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277198 2024-12-09T08:17:57,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277192 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277198 2024-12-09T08:17:57,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277198) roll requested 2024-12-09T08:17:57,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277205 2024-12-09T08:17:57,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277198 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277205 2024-12-09T08:17:57,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,213 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277205) roll requested 2024-12-09T08:17:57,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277213 2024-12-09T08:17:57,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277213 2024-12-09T08:17:57,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,221 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277213) roll requested 2024-12-09T08:17:57,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277221 2024-12-09T08:17:57,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277221 2024-12-09T08:17:57,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277221) roll requested 2024-12-09T08:17:57,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277230 2024-12-09T08:17:57,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277221 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277230 2024-12-09T08:17:57,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277230) roll requested 2024-12-09T08:17:57,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277238 2024-12-09T08:17:57,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277238 2024-12-09T08:17:57,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277238) roll requested 2024-12-09T08:17:57,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277246 2024-12-09T08:17:57,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277246 2024-12-09T08:17:57,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,254 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277246) roll requested 2024-12-09T08:17:57,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277254 2024-12-09T08:17:57,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277254 2024-12-09T08:17:57,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277254) roll requested 2024-12-09T08:17:57,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277263 2024-12-09T08:17:57,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277254 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277263 2024-12-09T08:17:57,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,271 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277263) roll requested 2024-12-09T08:17:57,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277271 2024-12-09T08:17:57,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277271 2024-12-09T08:17:57,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277271) roll requested 2024-12-09T08:17:57,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277285 2024-12-09T08:17:57,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277285 2024-12-09T08:17:57,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277285) roll requested 2024-12-09T08:17:57,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277292 2024-12-09T08:17:57,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277292 2024-12-09T08:17:57,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,297 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277292) roll requested 2024-12-09T08:17:57,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277298 2024-12-09T08:17:57,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277298 2024-12-09T08:17:57,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277298) roll requested 2024-12-09T08:17:57,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277306 2024-12-09T08:17:57,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277306 2024-12-09T08:17:57,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277306) roll requested 2024-12-09T08:17:57,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277312 2024-12-09T08:17:57,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277312 2024-12-09T08:17:57,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277312) roll requested 2024-12-09T08:17:57,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277317 2024-12-09T08:17:57,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277317 2024-12-09T08:17:57,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,325 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277317) roll requested 2024-12-09T08:17:57,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277325 2024-12-09T08:17:57,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277325 2024-12-09T08:17:57,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,330 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277325) roll requested 2024-12-09T08:17:57,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277330 2024-12-09T08:17:57,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277330 2024-12-09T08:17:57,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277330) roll requested 2024-12-09T08:17:57,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277335 2024-12-09T08:17:57,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277330 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277335 2024-12-09T08:17:57,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277335) roll requested 2024-12-09T08:17:57,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277339 2024-12-09T08:17:57,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277339 2024-12-09T08:17:57,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277339) roll requested 2024-12-09T08:17:57,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277345 2024-12-09T08:17:57,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277345 2024-12-09T08:17:57,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,351 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277345) roll requested 2024-12-09T08:17:57,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277351 2024-12-09T08:17:57,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277351 2024-12-09T08:17:57,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,359 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277351) roll requested 2024-12-09T08:17:57,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277359 2024-12-09T08:17:57,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277351 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277359 2024-12-09T08:17:57,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,366 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277359) roll requested 2024-12-09T08:17:57,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277366 2024-12-09T08:17:57,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277366 2024-12-09T08:17:57,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277366) roll requested 2024-12-09T08:17:57,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277375 2024-12-09T08:17:57,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277366 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277375 2024-12-09T08:17:57,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277375) roll requested 2024-12-09T08:17:57,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277380 2024-12-09T08:17:57,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277375 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277380 2024-12-09T08:17:57,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277380) roll requested 2024-12-09T08:17:57,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277384 2024-12-09T08:17:57,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277384 2024-12-09T08:17:57,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,390 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277384) roll requested 2024-12-09T08:17:57,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277390 2024-12-09T08:17:57,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277390 2024-12-09T08:17:57,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277390) roll requested 2024-12-09T08:17:57,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277396 2024-12-09T08:17:57,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277396 2024-12-09T08:17:57,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,402 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277396) roll requested 2024-12-09T08:17:57,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277403 2024-12-09T08:17:57,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277403 2024-12-09T08:17:57,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277403) roll requested 2024-12-09T08:17:57,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277411 2024-12-09T08:17:57,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277403 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277411 2024-12-09T08:17:57,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,418 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277411) roll requested 2024-12-09T08:17:57,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277418 2024-12-09T08:17:57,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277418 2024-12-09T08:17:57,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277418) roll requested 2024-12-09T08:17:57,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277426 2024-12-09T08:17:57,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277418 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277426 2024-12-09T08:17:57,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277426) roll requested 2024-12-09T08:17:57,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277434 2024-12-09T08:17:57,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277426 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277434 2024-12-09T08:17:57,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,442 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277434) roll requested 2024-12-09T08:17:57,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277442 2024-12-09T08:17:57,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277442 2024-12-09T08:17:57,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277442) roll requested 2024-12-09T08:17:57,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277452 2024-12-09T08:17:57,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277442 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277452 2024-12-09T08:17:57,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277452) roll requested 2024-12-09T08:17:57,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277461 2024-12-09T08:17:57,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277461 2024-12-09T08:17:57,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277461) roll requested 2024-12-09T08:17:57,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277466 2024-12-09T08:17:57,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277466 2024-12-09T08:17:57,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277466) roll requested 2024-12-09T08:17:57,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277471 2024-12-09T08:17:57,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277471 2024-12-09T08:17:57,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277471) roll requested 2024-12-09T08:17:57,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277476 2024-12-09T08:17:57,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277476 2024-12-09T08:17:57,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277476) roll requested 2024-12-09T08:17:57,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277481 2024-12-09T08:17:57,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277481 2024-12-09T08:17:57,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277481) roll requested 2024-12-09T08:17:57,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277487 2024-12-09T08:17:57,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277487 2024-12-09T08:17:57,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,497 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277487) roll requested 2024-12-09T08:17:57,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277498 2024-12-09T08:17:57,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277498 2024-12-09T08:17:57,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,505 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277498) roll requested 2024-12-09T08:17:57,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277505 2024-12-09T08:17:57,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277498 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277505 2024-12-09T08:17:57,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,511 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277505) roll requested 2024-12-09T08:17:57,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277512 2024-12-09T08:17:57,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277512 2024-12-09T08:17:57,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277512) roll requested 2024-12-09T08:17:57,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277519 2024-12-09T08:17:57,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277519 2024-12-09T08:17:57,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277519) roll requested 2024-12-09T08:17:57,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277527 2024-12-09T08:17:57,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277527 2024-12-09T08:17:57,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277527) roll requested 2024-12-09T08:17:57,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277536 2024-12-09T08:17:57,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277536 2024-12-09T08:17:57,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277536) roll requested 2024-12-09T08:17:57,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277543 2024-12-09T08:17:57,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277543 2024-12-09T08:17:57,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277543) roll requested 2024-12-09T08:17:57,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277552 2024-12-09T08:17:57,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277552 2024-12-09T08:17:57,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277552) roll requested 2024-12-09T08:17:57,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277559 2024-12-09T08:17:57,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277559 2024-12-09T08:17:57,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,567 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277559) roll requested 2024-12-09T08:17:57,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277567 2024-12-09T08:17:57,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277567 2024-12-09T08:17:57,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277567) roll requested 2024-12-09T08:17:57,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277577 2024-12-09T08:17:57,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277567 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277577 2024-12-09T08:17:57,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277577) roll requested 2024-12-09T08:17:57,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277583 2024-12-09T08:17:57,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277583 2024-12-09T08:17:57,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,590 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277583) roll requested 2024-12-09T08:17:57,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277591 2024-12-09T08:17:57,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277583 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277591 2024-12-09T08:17:57,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277591) roll requested 2024-12-09T08:17:57,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277599 2024-12-09T08:17:57,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277591 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277599 2024-12-09T08:17:57,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277599) roll requested 2024-12-09T08:17:57,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277609 2024-12-09T08:17:57,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277609 2024-12-09T08:17:57,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277609) roll requested 2024-12-09T08:17:57,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277617 2024-12-09T08:17:57,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277617 2024-12-09T08:17:57,622 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,622 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,622 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277617) roll requested 2024-12-09T08:17:57,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277622 2024-12-09T08:17:57,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277622 2024-12-09T08:17:57,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277622) roll requested 2024-12-09T08:17:57,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277627 2024-12-09T08:17:57,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277622 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277627 2024-12-09T08:17:57,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277627) roll requested 2024-12-09T08:17:57,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277632 2024-12-09T08:17:57,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277627 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277632 2024-12-09T08:17:57,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277632) roll requested 2024-12-09T08:17:57,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277636 2024-12-09T08:17:57,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277636 2024-12-09T08:17:57,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,643 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277636) roll requested 2024-12-09T08:17:57,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277643 2024-12-09T08:17:57,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277636 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277643 2024-12-09T08:17:57,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277643) roll requested 2024-12-09T08:17:57,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277650 2024-12-09T08:17:57,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277643 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277650 2024-12-09T08:17:57,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277650) roll requested 2024-12-09T08:17:57,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277656 2024-12-09T08:17:57,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277650 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277656 2024-12-09T08:17:57,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277656) roll requested 2024-12-09T08:17:57,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277662 2024-12-09T08:17:57,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277662 2024-12-09T08:17:57,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277662) roll requested 2024-12-09T08:17:57,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277667 2024-12-09T08:17:57,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277667 2024-12-09T08:17:57,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277667) roll requested 2024-12-09T08:17:57,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277672 2024-12-09T08:17:57,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277672 2024-12-09T08:17:57,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277672) roll requested 2024-12-09T08:17:57,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277678 2024-12-09T08:17:57,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277678 2024-12-09T08:17:57,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277678) roll requested 2024-12-09T08:17:57,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277683 2024-12-09T08:17:57,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277683 2024-12-09T08:17:57,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277683) roll requested 2024-12-09T08:17:57,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277690 2024-12-09T08:17:57,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277690 2024-12-09T08:17:57,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277690) roll requested 2024-12-09T08:17:57,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277696 2024-12-09T08:17:57,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277696 2024-12-09T08:17:57,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,702 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277696) roll requested 2024-12-09T08:17:57,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277702 2024-12-09T08:17:57,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277696 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277702 2024-12-09T08:17:57,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277702) roll requested 2024-12-09T08:17:57,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277709 2024-12-09T08:17:57,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277709 2024-12-09T08:17:57,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,716 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277709) roll requested 2024-12-09T08:17:57,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277716 2024-12-09T08:17:57,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277716 2024-12-09T08:17:57,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277716) roll requested 2024-12-09T08:17:57,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277725 2024-12-09T08:17:57,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277716 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277725 2024-12-09T08:17:57,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277725) roll requested 2024-12-09T08:17:57,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277733 2024-12-09T08:17:57,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277733 2024-12-09T08:17:57,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,741 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277733) roll requested 2024-12-09T08:17:57,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277741 2024-12-09T08:17:57,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277741 2024-12-09T08:17:57,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,746 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277741) roll requested 2024-12-09T08:17:57,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277746 2024-12-09T08:17:57,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277741 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277746 2024-12-09T08:17:57,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277746) roll requested 2024-12-09T08:17:57,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277751 2024-12-09T08:17:57,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277751 2024-12-09T08:17:57,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,757 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277751) roll requested 2024-12-09T08:17:57,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277757 2024-12-09T08:17:57,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277757 2024-12-09T08:17:57,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277757) roll requested 2024-12-09T08:17:57,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277763 2024-12-09T08:17:57,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277763 2024-12-09T08:17:57,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277763) roll requested 2024-12-09T08:17:57,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277768 2024-12-09T08:17:57,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277763 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277768 2024-12-09T08:17:57,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,773 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277768) roll requested 2024-12-09T08:17:57,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277773 2024-12-09T08:17:57,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277773 2024-12-09T08:17:57,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,779 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277773) roll requested 2024-12-09T08:17:57,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277779 2024-12-09T08:17:57,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277779 2024-12-09T08:17:57,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277779) roll requested 2024-12-09T08:17:57,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277784 2024-12-09T08:17:57,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277784 2024-12-09T08:17:57,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277784) roll requested 2024-12-09T08:17:57,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277789 2024-12-09T08:17:57,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277789 2024-12-09T08:17:57,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,794 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277789) roll requested 2024-12-09T08:17:57,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277794 2024-12-09T08:17:57,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277794 2024-12-09T08:17:57,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277794) roll requested 2024-12-09T08:17:57,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277799 2024-12-09T08:17:57,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277794 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277799 2024-12-09T08:17:57,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277799) roll requested 2024-12-09T08:17:57,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277804 2024-12-09T08:17:57,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277804 2024-12-09T08:17:57,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,810 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277804) roll requested 2024-12-09T08:17:57,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277810 2024-12-09T08:17:57,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277810 2024-12-09T08:17:57,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277810) roll requested 2024-12-09T08:17:57,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277817 2024-12-09T08:17:57,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277810 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277817 2024-12-09T08:17:57,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,824 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277817) roll requested 2024-12-09T08:17:57,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277824 2024-12-09T08:17:57,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277824 2024-12-09T08:17:57,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277824) roll requested 2024-12-09T08:17:57,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277831 2024-12-09T08:17:57,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277824 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277831 2024-12-09T08:17:57,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,838 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277831) roll requested 2024-12-09T08:17:57,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277838 2024-12-09T08:17:57,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277831 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277838 2024-12-09T08:17:57,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277838) roll requested 2024-12-09T08:17:57,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277847 2024-12-09T08:17:57,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277838 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277847 2024-12-09T08:17:57,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277847) roll requested 2024-12-09T08:17:57,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277855 2024-12-09T08:17:57,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277855 2024-12-09T08:17:57,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277855) roll requested 2024-12-09T08:17:57,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277862 2024-12-09T08:17:57,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277855 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277862 2024-12-09T08:17:57,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,871 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277862) roll requested 2024-12-09T08:17:57,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277871 2024-12-09T08:17:57,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277871 2024-12-09T08:17:57,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277871) roll requested 2024-12-09T08:17:57,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277881 2024-12-09T08:17:57,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277881 2024-12-09T08:17:57,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,889 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277881) roll requested 2024-12-09T08:17:57,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277889 2024-12-09T08:17:57,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277889 2024-12-09T08:17:57,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277889) roll requested 2024-12-09T08:17:57,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277895 2024-12-09T08:17:57,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277895 2024-12-09T08:17:57,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277895) roll requested 2024-12-09T08:17:57,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277902 2024-12-09T08:17:57,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277902 2024-12-09T08:17:57,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277902) roll requested 2024-12-09T08:17:57,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277908 2024-12-09T08:17:57,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277902 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277908 2024-12-09T08:17:57,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,915 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277908) roll requested 2024-12-09T08:17:57,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277915 2024-12-09T08:17:57,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277915 2024-12-09T08:17:57,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,922 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277915) roll requested 2024-12-09T08:17:57,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277923 2024-12-09T08:17:57,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277923 2024-12-09T08:17:57,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,929 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277923) roll requested 2024-12-09T08:17:57,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277929 2024-12-09T08:17:57,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277923 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277929 2024-12-09T08:17:57,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,938 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277929) roll requested 2024-12-09T08:17:57,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277938 2024-12-09T08:17:57,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277938 2024-12-09T08:17:57,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277938) roll requested 2024-12-09T08:17:57,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277948 2024-12-09T08:17:57,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277938 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277948 2024-12-09T08:17:57,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277948) roll requested 2024-12-09T08:17:57,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277961 2024-12-09T08:17:57,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277948 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277961 2024-12-09T08:17:57,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,970 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277961) roll requested 2024-12-09T08:17:57,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277970 2024-12-09T08:17:57,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277970 2024-12-09T08:17:57,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,975 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277970) roll requested 2024-12-09T08:17:57,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277975 2024-12-09T08:17:57,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277970 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277975 2024-12-09T08:17:57,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,979 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277975) roll requested 2024-12-09T08:17:57,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277979 2024-12-09T08:17:57,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277975 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277979 2024-12-09T08:17:57,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277979) roll requested 2024-12-09T08:17:57,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277986 2024-12-09T08:17:57,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277979 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277986 2024-12-09T08:17:57,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277986) roll requested 2024-12-09T08:17:57,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277992 2024-12-09T08:17:57,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:57,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277986 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277992 2024-12-09T08:17:57,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:57,999 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:57,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277992) roll requested 2024-12-09T08:17:57,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732277999 2024-12-09T08:17:58,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277999 2024-12-09T08:17:58,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732277999) roll requested 2024-12-09T08:17:58,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278007 2024-12-09T08:17:58,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732277999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278007 2024-12-09T08:17:58,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,015 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278007) roll requested 2024-12-09T08:17:58,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278015 2024-12-09T08:17:58,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278015 2024-12-09T08:17:58,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278015) roll requested 2024-12-09T08:17:58,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278023 2024-12-09T08:17:58,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278015 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278023 2024-12-09T08:17:58,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278023) roll requested 2024-12-09T08:17:58,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278030 2024-12-09T08:17:58,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278030 2024-12-09T08:17:58,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,039 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278030) roll requested 2024-12-09T08:17:58,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278039 2024-12-09T08:17:58,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278030 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278039 2024-12-09T08:17:58,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278039) roll requested 2024-12-09T08:17:58,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278047 2024-12-09T08:17:58,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278039 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278047 2024-12-09T08:17:58,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278047) roll requested 2024-12-09T08:17:58,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278055 2024-12-09T08:17:58,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278055 2024-12-09T08:17:58,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278055) roll requested 2024-12-09T08:17:58,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278064 2024-12-09T08:17:58,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278064 2024-12-09T08:17:58,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278064) roll requested 2024-12-09T08:17:58,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278072 2024-12-09T08:17:58,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278072 2024-12-09T08:17:58,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,079 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278072) roll requested 2024-12-09T08:17:58,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278079 2024-12-09T08:17:58,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278079 2024-12-09T08:17:58,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,086 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278079) roll requested 2024-12-09T08:17:58,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278086 2024-12-09T08:17:58,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278079 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278086 2024-12-09T08:17:58,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,094 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278086) roll requested 2024-12-09T08:17:58,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278094 2024-12-09T08:17:58,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278086 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278094 2024-12-09T08:17:58,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,102 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278094) roll requested 2024-12-09T08:17:58,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278102 2024-12-09T08:17:58,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278094 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278102 2024-12-09T08:17:58,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,110 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278102) roll requested 2024-12-09T08:17:58,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278110 2024-12-09T08:17:58,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278102 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278110 2024-12-09T08:17:58,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278110) roll requested 2024-12-09T08:17:58,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278120 2024-12-09T08:17:58,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278110 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278120 2024-12-09T08:17:58,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,126 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278120) roll requested 2024-12-09T08:17:58,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278127 2024-12-09T08:17:58,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278127 2024-12-09T08:17:58,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278127) roll requested 2024-12-09T08:17:58,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278135 2024-12-09T08:17:58,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278127 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278135 2024-12-09T08:17:58,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278135) roll requested 2024-12-09T08:17:58,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278142 2024-12-09T08:17:58,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278142 2024-12-09T08:17:58,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278142) roll requested 2024-12-09T08:17:58,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278148 2024-12-09T08:17:58,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278148 2024-12-09T08:17:58,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,154 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278148) roll requested 2024-12-09T08:17:58,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278155 2024-12-09T08:17:58,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278155 2024-12-09T08:17:58,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,163 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278155) roll requested 2024-12-09T08:17:58,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278163 2024-12-09T08:17:58,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278155 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278163 2024-12-09T08:17:58,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278163) roll requested 2024-12-09T08:17:58,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278172 2024-12-09T08:17:58,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278163 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278172 2024-12-09T08:17:58,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278172) roll requested 2024-12-09T08:17:58,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278181 2024-12-09T08:17:58,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278181 2024-12-09T08:17:58,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,190 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278181) roll requested 2024-12-09T08:17:58,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278190 2024-12-09T08:17:58,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278190 2024-12-09T08:17:58,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,198 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278190) roll requested 2024-12-09T08:17:58,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278198 2024-12-09T08:17:58,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278198 2024-12-09T08:17:58,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278198) roll requested 2024-12-09T08:17:58,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278206 2024-12-09T08:17:58,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278198 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278206 2024-12-09T08:17:58,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278206) roll requested 2024-12-09T08:17:58,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278214 2024-12-09T08:17:58,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278206 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278214 2024-12-09T08:17:58,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278214) roll requested 2024-12-09T08:17:58,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278222 2024-12-09T08:17:58,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278222 2024-12-09T08:17:58,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278222) roll requested 2024-12-09T08:17:58,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278231 2024-12-09T08:17:58,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278231 2024-12-09T08:17:58,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278231) roll requested 2024-12-09T08:17:58,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278238 2024-12-09T08:17:58,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278238 2024-12-09T08:17:58,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,243 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278238) roll requested 2024-12-09T08:17:58,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278243 2024-12-09T08:17:58,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278243 2024-12-09T08:17:58,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278243) roll requested 2024-12-09T08:17:58,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278249 2024-12-09T08:17:58,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278243 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278249 2024-12-09T08:17:58,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,258 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278249) roll requested 2024-12-09T08:17:58,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278258 2024-12-09T08:17:58,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278258 2024-12-09T08:17:58,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,266 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278258) roll requested 2024-12-09T08:17:58,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278267 2024-12-09T08:17:58,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278258 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278267 2024-12-09T08:17:58,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,276 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278267) roll requested 2024-12-09T08:17:58,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278276 2024-12-09T08:17:58,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278276 2024-12-09T08:17:58,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278276) roll requested 2024-12-09T08:17:58,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278286 2024-12-09T08:17:58,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278276 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278286 2024-12-09T08:17:58,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278286) roll requested 2024-12-09T08:17:58,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278294 2024-12-09T08:17:58,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278294 2024-12-09T08:17:58,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278294) roll requested 2024-12-09T08:17:58,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278302 2024-12-09T08:17:58,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278302 2024-12-09T08:17:58,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,309 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278302) roll requested 2024-12-09T08:17:58,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278310 2024-12-09T08:17:58,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278302 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278310 2024-12-09T08:17:58,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,319 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278310) roll requested 2024-12-09T08:17:58,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278319 2024-12-09T08:17:58,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278319 2024-12-09T08:17:58,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278319) roll requested 2024-12-09T08:17:58,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278327 2024-12-09T08:17:58,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278319 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278327 2024-12-09T08:17:58,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,333 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278327) roll requested 2024-12-09T08:17:58,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278333 2024-12-09T08:17:58,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278333 2024-12-09T08:17:58,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,338 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278333) roll requested 2024-12-09T08:17:58,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278339 2024-12-09T08:17:58,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278339 2024-12-09T08:17:58,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,346 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278339) roll requested 2024-12-09T08:17:58,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278346 2024-12-09T08:17:58,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278346 2024-12-09T08:17:58,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278346) roll requested 2024-12-09T08:17:58,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278354 2024-12-09T08:17:58,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278346 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278354 2024-12-09T08:17:58,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278354) roll requested 2024-12-09T08:17:58,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278363 2024-12-09T08:17:58,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278354 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278363 2024-12-09T08:17:58,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278363) roll requested 2024-12-09T08:17:58,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278369 2024-12-09T08:17:58,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278369 2024-12-09T08:17:58,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278369) roll requested 2024-12-09T08:17:58,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278375 2024-12-09T08:17:58,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278375 2024-12-09T08:17:58,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278375) roll requested 2024-12-09T08:17:58,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278382 2024-12-09T08:17:58,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278375 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278382 2024-12-09T08:17:58,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,390 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278382) roll requested 2024-12-09T08:17:58,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278390 2024-12-09T08:17:58,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278390 2024-12-09T08:17:58,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,397 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278390) roll requested 2024-12-09T08:17:58,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278397 2024-12-09T08:17:58,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278397 2024-12-09T08:17:58,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278397) roll requested 2024-12-09T08:17:58,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278405 2024-12-09T08:17:58,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278405 2024-12-09T08:17:58,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278405) roll requested 2024-12-09T08:17:58,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278412 2024-12-09T08:17:58,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278405 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278412 2024-12-09T08:17:58,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278412) roll requested 2024-12-09T08:17:58,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278420 2024-12-09T08:17:58,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278420 2024-12-09T08:17:58,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278420) roll requested 2024-12-09T08:17:58,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278428 2024-12-09T08:17:58,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278420 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278428 2024-12-09T08:17:58,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,435 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278428) roll requested 2024-12-09T08:17:58,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278436 2024-12-09T08:17:58,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278436 2024-12-09T08:17:58,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,444 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278436) roll requested 2024-12-09T08:17:58,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278444 2024-12-09T08:17:58,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278436 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278444 2024-12-09T08:17:58,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278444) roll requested 2024-12-09T08:17:58,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278452 2024-12-09T08:17:58,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278444 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278452 2024-12-09T08:17:58,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278452) roll requested 2024-12-09T08:17:58,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278459 2024-12-09T08:17:58,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278459 2024-12-09T08:17:58,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278459) roll requested 2024-12-09T08:17:58,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278467 2024-12-09T08:17:58,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278467 2024-12-09T08:17:58,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278467) roll requested 2024-12-09T08:17:58,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278475 2024-12-09T08:17:58,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278475 2024-12-09T08:17:58,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278475) roll requested 2024-12-09T08:17:58,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278481 2024-12-09T08:17:58,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278475 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278481 2024-12-09T08:17:58,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278481) roll requested 2024-12-09T08:17:58,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278488 2024-12-09T08:17:58,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278488 2024-12-09T08:17:58,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278488) roll requested 2024-12-09T08:17:58,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278496 2024-12-09T08:17:58,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278496 2024-12-09T08:17:58,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278496) roll requested 2024-12-09T08:17:58,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278504 2024-12-09T08:17:58,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278504 2024-12-09T08:17:58,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278504) roll requested 2024-12-09T08:17:58,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278513 2024-12-09T08:17:58,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278504 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278513 2024-12-09T08:17:58,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278513) roll requested 2024-12-09T08:17:58,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278522 2024-12-09T08:17:58,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278522 2024-12-09T08:17:58,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278522) roll requested 2024-12-09T08:17:58,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278530 2024-12-09T08:17:58,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278530 2024-12-09T08:17:58,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,538 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278530) roll requested 2024-12-09T08:17:58,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278538 2024-12-09T08:17:58,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278538 2024-12-09T08:17:58,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278538) roll requested 2024-12-09T08:17:58,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278547 2024-12-09T08:17:58,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278538 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278547 2024-12-09T08:17:58,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278547) roll requested 2024-12-09T08:17:58,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278556 2024-12-09T08:17:58,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278556 2024-12-09T08:17:58,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278556) roll requested 2024-12-09T08:17:58,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278566 2024-12-09T08:17:58,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278566 2024-12-09T08:17:58,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,575 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278566) roll requested 2024-12-09T08:17:58,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278576 2024-12-09T08:17:58,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278566 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278576 2024-12-09T08:17:58,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278576) roll requested 2024-12-09T08:17:58,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278584 2024-12-09T08:17:58,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278584 2024-12-09T08:17:58,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278584) roll requested 2024-12-09T08:17:58,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278592 2024-12-09T08:17:58,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278592 2024-12-09T08:17:58,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278592) roll requested 2024-12-09T08:17:58,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278601 2024-12-09T08:17:58,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278601 2024-12-09T08:17:58,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278601) roll requested 2024-12-09T08:17:58,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278609 2024-12-09T08:17:58,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278609 2024-12-09T08:17:58,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,615 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278609) roll requested 2024-12-09T08:17:58,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278615 2024-12-09T08:17:58,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278615 2024-12-09T08:17:58,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278615) roll requested 2024-12-09T08:17:58,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278624 2024-12-09T08:17:58,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278615 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278624 2024-12-09T08:17:58,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,633 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278624) roll requested 2024-12-09T08:17:58,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278634 2024-12-09T08:17:58,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278624 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278634 2024-12-09T08:17:58,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,643 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278634) roll requested 2024-12-09T08:17:58,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278644 2024-12-09T08:17:58,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278634 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278644 2024-12-09T08:17:58,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278644) roll requested 2024-12-09T08:17:58,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278654 2024-12-09T08:17:58,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278654 2024-12-09T08:17:58,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,663 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278654) roll requested 2024-12-09T08:17:58,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278664 2024-12-09T08:17:58,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278664 2024-12-09T08:17:58,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,671 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278664) roll requested 2024-12-09T08:17:58,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278671 2024-12-09T08:17:58,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278671 2024-12-09T08:17:58,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,678 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278671) roll requested 2024-12-09T08:17:58,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278678 2024-12-09T08:17:58,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278671 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278678 2024-12-09T08:17:58,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278678) roll requested 2024-12-09T08:17:58,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278685 2024-12-09T08:17:58,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278685 2024-12-09T08:17:58,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278685) roll requested 2024-12-09T08:17:58,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278694 2024-12-09T08:17:58,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278694 2024-12-09T08:17:58,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278694) roll requested 2024-12-09T08:17:58,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278701 2024-12-09T08:17:58,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278694 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278701 2024-12-09T08:17:58,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278701) roll requested 2024-12-09T08:17:58,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278708 2024-12-09T08:17:58,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278708 2024-12-09T08:17:58,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,716 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278708) roll requested 2024-12-09T08:17:58,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278717 2024-12-09T08:17:58,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278717 2024-12-09T08:17:58,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278717) roll requested 2024-12-09T08:17:58,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278725 2024-12-09T08:17:58,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278717 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278725 2024-12-09T08:17:58,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278725) roll requested 2024-12-09T08:17:58,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278733 2024-12-09T08:17:58,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278733 2024-12-09T08:17:58,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278733) roll requested 2024-12-09T08:17:58,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278742 2024-12-09T08:17:58,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278742 2024-12-09T08:17:58,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278742) roll requested 2024-12-09T08:17:58,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278750 2024-12-09T08:17:58,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278750 2024-12-09T08:17:58,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278750) roll requested 2024-12-09T08:17:58,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278759 2024-12-09T08:17:58,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278759 2024-12-09T08:17:58,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278759) roll requested 2024-12-09T08:17:58,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278768 2024-12-09T08:17:58,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278768 2024-12-09T08:17:58,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278768) roll requested 2024-12-09T08:17:58,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278777 2024-12-09T08:17:58,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278777 2024-12-09T08:17:58,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,787 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278777) roll requested 2024-12-09T08:17:58,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278787 2024-12-09T08:17:58,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278787 2024-12-09T08:17:58,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278787) roll requested 2024-12-09T08:17:58,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278797 2024-12-09T08:17:58,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278787 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278797 2024-12-09T08:17:58,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278797) roll requested 2024-12-09T08:17:58,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278805 2024-12-09T08:17:58,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278805 2024-12-09T08:17:58,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,814 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278805) roll requested 2024-12-09T08:17:58,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278814 2024-12-09T08:17:58,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278814 2024-12-09T08:17:58,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278814) roll requested 2024-12-09T08:17:58,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278822 2024-12-09T08:17:58,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278822 2024-12-09T08:17:58,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,830 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278822) roll requested 2024-12-09T08:17:58,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278830 2024-12-09T08:17:58,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278830 2024-12-09T08:17:58,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,842 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278830) roll requested 2024-12-09T08:17:58,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278843 2024-12-09T08:17:58,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278830 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278843 2024-12-09T08:17:58,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,850 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278843) roll requested 2024-12-09T08:17:58,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278850 2024-12-09T08:17:58,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278843 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278850 2024-12-09T08:17:58,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,858 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278850) roll requested 2024-12-09T08:17:58,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278858 2024-12-09T08:17:58,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278858 2024-12-09T08:17:58,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278858) roll requested 2024-12-09T08:17:58,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278867 2024-12-09T08:17:58,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278858 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278867 2024-12-09T08:17:58,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278867) roll requested 2024-12-09T08:17:58,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278877 2024-12-09T08:17:58,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278877 2024-12-09T08:17:58,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,886 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278877) roll requested 2024-12-09T08:17:58,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278886 2024-12-09T08:17:58,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278877 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278886 2024-12-09T08:17:58,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278886) roll requested 2024-12-09T08:17:58,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278895 2024-12-09T08:17:58,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278886 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278895 2024-12-09T08:17:58,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278895) roll requested 2024-12-09T08:17:58,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278902 2024-12-09T08:17:58,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278902 2024-12-09T08:17:58,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,909 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278902) roll requested 2024-12-09T08:17:58,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278909 2024-12-09T08:17:58,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278902 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278909 2024-12-09T08:17:58,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,919 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278909) roll requested 2024-12-09T08:17:58,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278919 2024-12-09T08:17:58,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278919 2024-12-09T08:17:58,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278919) roll requested 2024-12-09T08:17:58,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278928 2024-12-09T08:17:58,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278928 2024-12-09T08:17:58,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278928) roll requested 2024-12-09T08:17:58,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278937 2024-12-09T08:17:58,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278937 2024-12-09T08:17:58,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278937) roll requested 2024-12-09T08:17:58,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278944 2024-12-09T08:17:58,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278944 2024-12-09T08:17:58,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,950 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278944) roll requested 2024-12-09T08:17:58,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278950 2024-12-09T08:17:58,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278950 2024-12-09T08:17:58,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278950) roll requested 2024-12-09T08:17:58,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278959 2024-12-09T08:17:58,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278950 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278959 2024-12-09T08:17:58,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,967 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278959) roll requested 2024-12-09T08:17:58,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278967 2024-12-09T08:17:58,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278959 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278967 2024-12-09T08:17:58,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,975 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278967) roll requested 2024-12-09T08:17:58,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278975 2024-12-09T08:17:58,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278967 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278975 2024-12-09T08:17:58,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278975) roll requested 2024-12-09T08:17:58,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278983 2024-12-09T08:17:58,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278975 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278983 2024-12-09T08:17:58,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278983) roll requested 2024-12-09T08:17:58,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278991 2024-12-09T08:17:58,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:58,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278991 2024-12-09T08:17:58,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:58,999 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:58,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278991) roll requested 2024-12-09T08:17:58,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732278999 2024-12-09T08:17:59,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278999 2024-12-09T08:17:59,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732278999) roll requested 2024-12-09T08:17:59,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279008 2024-12-09T08:17:59,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732278999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279008 2024-12-09T08:17:59,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279008) roll requested 2024-12-09T08:17:59,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279017 2024-12-09T08:17:59,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279017 2024-12-09T08:17:59,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,027 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279017) roll requested 2024-12-09T08:17:59,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279027 2024-12-09T08:17:59,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279027 2024-12-09T08:17:59,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279027) roll requested 2024-12-09T08:17:59,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279037 2024-12-09T08:17:59,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279027 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279037 2024-12-09T08:17:59,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,045 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279037) roll requested 2024-12-09T08:17:59,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279045 2024-12-09T08:17:59,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279045 2024-12-09T08:17:59,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279045) roll requested 2024-12-09T08:17:59,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279053 2024-12-09T08:17:59,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279045 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279053 2024-12-09T08:17:59,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279053) roll requested 2024-12-09T08:17:59,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279061 2024-12-09T08:17:59,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279053 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279061 2024-12-09T08:17:59,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279061) roll requested 2024-12-09T08:17:59,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279068 2024-12-09T08:17:59,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279061 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279068 2024-12-09T08:17:59,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,074 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279068) roll requested 2024-12-09T08:17:59,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279074 2024-12-09T08:17:59,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279074 2024-12-09T08:17:59,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279074) roll requested 2024-12-09T08:17:59,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279080 2024-12-09T08:17:59,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279074 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279080 2024-12-09T08:17:59,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279080) roll requested 2024-12-09T08:17:59,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279088 2024-12-09T08:17:59,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279088 2024-12-09T08:17:59,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279088) roll requested 2024-12-09T08:17:59,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279097 2024-12-09T08:17:59,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279088 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279097 2024-12-09T08:17:59,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279097) roll requested 2024-12-09T08:17:59,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279106 2024-12-09T08:17:59,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279106 2024-12-09T08:17:59,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279106) roll requested 2024-12-09T08:17:59,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279112 2024-12-09T08:17:59,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279112 2024-12-09T08:17:59,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279112) roll requested 2024-12-09T08:17:59,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279119 2024-12-09T08:17:59,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279119 2024-12-09T08:17:59,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279119) roll requested 2024-12-09T08:17:59,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279127 2024-12-09T08:17:59,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279119 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279127 2024-12-09T08:17:59,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279127) roll requested 2024-12-09T08:17:59,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279136 2024-12-09T08:17:59,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279127 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279136 2024-12-09T08:17:59,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279136) roll requested 2024-12-09T08:17:59,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279143 2024-12-09T08:17:59,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279143 2024-12-09T08:17:59,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,149 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279143) roll requested 2024-12-09T08:17:59,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279150 2024-12-09T08:17:59,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279150 2024-12-09T08:17:59,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279150) roll requested 2024-12-09T08:17:59,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279157 2024-12-09T08:17:59,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279157 2024-12-09T08:17:59,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,166 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279157) roll requested 2024-12-09T08:17:59,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279166 2024-12-09T08:17:59,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279166 2024-12-09T08:17:59,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,175 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279166) roll requested 2024-12-09T08:17:59,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279175 2024-12-09T08:17:59,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279166 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279175 2024-12-09T08:17:59,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279175) roll requested 2024-12-09T08:17:59,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279183 2024-12-09T08:17:59,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279175 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279183 2024-12-09T08:17:59,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279183) roll requested 2024-12-09T08:17:59,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279189 2024-12-09T08:17:59,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279189 2024-12-09T08:17:59,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279189) roll requested 2024-12-09T08:17:59,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279197 2024-12-09T08:17:59,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279197 2024-12-09T08:17:59,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279197) roll requested 2024-12-09T08:17:59,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279205 2024-12-09T08:17:59,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279205 2024-12-09T08:17:59,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279205) roll requested 2024-12-09T08:17:59,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279210 2024-12-09T08:17:59,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279210 2024-12-09T08:17:59,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279210) roll requested 2024-12-09T08:17:59,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279216 2024-12-09T08:17:59,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279210 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279216 2024-12-09T08:17:59,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279216) roll requested 2024-12-09T08:17:59,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279222 2024-12-09T08:17:59,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279222 2024-12-09T08:17:59,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279222) roll requested 2024-12-09T08:17:59,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279230 2024-12-09T08:17:59,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279230 2024-12-09T08:17:59,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279230) roll requested 2024-12-09T08:17:59,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279238 2024-12-09T08:17:59,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279238 2024-12-09T08:17:59,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,247 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279238) roll requested 2024-12-09T08:17:59,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279248 2024-12-09T08:17:59,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279248 2024-12-09T08:17:59,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279248) roll requested 2024-12-09T08:17:59,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279257 2024-12-09T08:17:59,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279257 2024-12-09T08:17:59,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279257) roll requested 2024-12-09T08:17:59,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279267 2024-12-09T08:17:59,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279267 2024-12-09T08:17:59,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279267) roll requested 2024-12-09T08:17:59,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279275 2024-12-09T08:17:59,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279275 2024-12-09T08:17:59,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279275) roll requested 2024-12-09T08:17:59,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279282 2024-12-09T08:17:59,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279282 2024-12-09T08:17:59,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,290 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279282) roll requested 2024-12-09T08:17:59,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279290 2024-12-09T08:17:59,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279282 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279290 2024-12-09T08:17:59,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279290) roll requested 2024-12-09T08:17:59,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279299 2024-12-09T08:17:59,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279290 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279299 2024-12-09T08:17:59,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,309 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279299) roll requested 2024-12-09T08:17:59,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279310 2024-12-09T08:17:59,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279310 2024-12-09T08:17:59,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279310) roll requested 2024-12-09T08:17:59,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279319 2024-12-09T08:17:59,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279319 2024-12-09T08:17:59,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,326 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279319) roll requested 2024-12-09T08:17:59,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279326 2024-12-09T08:17:59,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279319 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279326 2024-12-09T08:17:59,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,333 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279326) roll requested 2024-12-09T08:17:59,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279333 2024-12-09T08:17:59,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279326 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279333 2024-12-09T08:17:59,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279333) roll requested 2024-12-09T08:17:59,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279341 2024-12-09T08:17:59,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279341 2024-12-09T08:17:59,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279341) roll requested 2024-12-09T08:17:59,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279349 2024-12-09T08:17:59,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279349 2024-12-09T08:17:59,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,358 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279349) roll requested 2024-12-09T08:17:59,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279358 2024-12-09T08:17:59,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279349 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279358 2024-12-09T08:17:59,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279358) roll requested 2024-12-09T08:17:59,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279363 2024-12-09T08:17:59,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279358 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279363 2024-12-09T08:17:59,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279363) roll requested 2024-12-09T08:17:59,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279369 2024-12-09T08:17:59,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279369 2024-12-09T08:17:59,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279369) roll requested 2024-12-09T08:17:59,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279377 2024-12-09T08:17:59,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279377 2024-12-09T08:17:59,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279377) roll requested 2024-12-09T08:17:59,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279385 2024-12-09T08:17:59,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279377 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279385 2024-12-09T08:17:59,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,393 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279385) roll requested 2024-12-09T08:17:59,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279393 2024-12-09T08:17:59,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279393 2024-12-09T08:17:59,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279393) roll requested 2024-12-09T08:17:59,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279400 2024-12-09T08:17:59,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279400 2024-12-09T08:17:59,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,408 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279400) roll requested 2024-12-09T08:17:59,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279408 2024-12-09T08:17:59,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279408 2024-12-09T08:17:59,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,417 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279408) roll requested 2024-12-09T08:17:59,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279417 2024-12-09T08:17:59,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279408 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279417 2024-12-09T08:17:59,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,423 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279417) roll requested 2024-12-09T08:17:59,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279424 2024-12-09T08:17:59,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279417 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279424 2024-12-09T08:17:59,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,430 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279424) roll requested 2024-12-09T08:17:59,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279430 2024-12-09T08:17:59,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279430 2024-12-09T08:17:59,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,438 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279430) roll requested 2024-12-09T08:17:59,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279438 2024-12-09T08:17:59,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279438 2024-12-09T08:17:59,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279438) roll requested 2024-12-09T08:17:59,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279447 2024-12-09T08:17:59,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279438 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279447 2024-12-09T08:17:59,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279447) roll requested 2024-12-09T08:17:59,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279456 2024-12-09T08:17:59,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279456 2024-12-09T08:17:59,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279456) roll requested 2024-12-09T08:17:59,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279466 2024-12-09T08:17:59,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279466 2024-12-09T08:17:59,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279466) roll requested 2024-12-09T08:17:59,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279474 2024-12-09T08:17:59,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279474 2024-12-09T08:17:59,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279474) roll requested 2024-12-09T08:17:59,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279482 2024-12-09T08:17:59,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279482 2024-12-09T08:17:59,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279482) roll requested 2024-12-09T08:17:59,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279489 2024-12-09T08:17:59,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279482 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279489 2024-12-09T08:17:59,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,497 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279489) roll requested 2024-12-09T08:17:59,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279497 2024-12-09T08:17:59,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279497 2024-12-09T08:17:59,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279497) roll requested 2024-12-09T08:17:59,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279504 2024-12-09T08:17:59,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279504 2024-12-09T08:17:59,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279504) roll requested 2024-12-09T08:17:59,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279512 2024-12-09T08:17:59,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279504 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279512 2024-12-09T08:17:59,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279512) roll requested 2024-12-09T08:17:59,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279522 2024-12-09T08:17:59,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279522 2024-12-09T08:17:59,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279522) roll requested 2024-12-09T08:17:59,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279533 2024-12-09T08:17:59,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279533 2024-12-09T08:17:59,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279533) roll requested 2024-12-09T08:17:59,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279539 2024-12-09T08:17:59,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279539 2024-12-09T08:17:59,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,545 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279539) roll requested 2024-12-09T08:17:59,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279545 2024-12-09T08:17:59,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279545 2024-12-09T08:17:59,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279545) roll requested 2024-12-09T08:17:59,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279549 2024-12-09T08:17:59,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279545 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279549 2024-12-09T08:17:59,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279549) roll requested 2024-12-09T08:17:59,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279553 2024-12-09T08:17:59,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279553 2024-12-09T08:17:59,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279553) roll requested 2024-12-09T08:17:59,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279559 2024-12-09T08:17:59,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279559 2024-12-09T08:17:59,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279559) roll requested 2024-12-09T08:17:59,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279567 2024-12-09T08:17:59,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279567 2024-12-09T08:17:59,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,574 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279567) roll requested 2024-12-09T08:17:59,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279574 2024-12-09T08:17:59,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279567 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279574 2024-12-09T08:17:59,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,582 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279574) roll requested 2024-12-09T08:17:59,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279582 2024-12-09T08:17:59,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279574 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279582 2024-12-09T08:17:59,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279582) roll requested 2024-12-09T08:17:59,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279591 2024-12-09T08:17:59,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279582 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279591 2024-12-09T08:17:59,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,600 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279591) roll requested 2024-12-09T08:17:59,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279601 2024-12-09T08:17:59,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279591 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279601 2024-12-09T08:17:59,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279601) roll requested 2024-12-09T08:17:59,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279611 2024-12-09T08:17:59,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279611 2024-12-09T08:17:59,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,619 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279611) roll requested 2024-12-09T08:17:59,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279620 2024-12-09T08:17:59,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279611 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279620 2024-12-09T08:17:59,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279620) roll requested 2024-12-09T08:17:59,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279628 2024-12-09T08:17:59,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279628 2024-12-09T08:17:59,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279628) roll requested 2024-12-09T08:17:59,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279637 2024-12-09T08:17:59,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279637 2024-12-09T08:17:59,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279637) roll requested 2024-12-09T08:17:59,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279643 2024-12-09T08:17:59,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279637 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279643 2024-12-09T08:17:59,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279643) roll requested 2024-12-09T08:17:59,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279648 2024-12-09T08:17:59,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279643 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279648 2024-12-09T08:17:59,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,655 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279648) roll requested 2024-12-09T08:17:59,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279655 2024-12-09T08:17:59,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279655 2024-12-09T08:17:59,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,663 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279655) roll requested 2024-12-09T08:17:59,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279663 2024-12-09T08:17:59,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279655 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279663 2024-12-09T08:17:59,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279663) roll requested 2024-12-09T08:17:59,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279670 2024-12-09T08:17:59,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279663 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279670 2024-12-09T08:17:59,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,679 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279670) roll requested 2024-12-09T08:17:59,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279679 2024-12-09T08:17:59,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279670 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279679 2024-12-09T08:17:59,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,687 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279679) roll requested 2024-12-09T08:17:59,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279687 2024-12-09T08:17:59,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279679 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279687 2024-12-09T08:17:59,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279687) roll requested 2024-12-09T08:17:59,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279696 2024-12-09T08:17:59,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279687 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279696 2024-12-09T08:17:59,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279696) roll requested 2024-12-09T08:17:59,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279704 2024-12-09T08:17:59,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279696 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279704 2024-12-09T08:17:59,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,710 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279704) roll requested 2024-12-09T08:17:59,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279710 2024-12-09T08:17:59,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279710 2024-12-09T08:17:59,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279710) roll requested 2024-12-09T08:17:59,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279719 2024-12-09T08:17:59,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279710 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279719 2024-12-09T08:17:59,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279719) roll requested 2024-12-09T08:17:59,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279726 2024-12-09T08:17:59,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279726 2024-12-09T08:17:59,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279726) roll requested 2024-12-09T08:17:59,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279734 2024-12-09T08:17:59,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279734 2024-12-09T08:17:59,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279734) roll requested 2024-12-09T08:17:59,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279744 2024-12-09T08:17:59,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279744 2024-12-09T08:17:59,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279744) roll requested 2024-12-09T08:17:59,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279750 2024-12-09T08:17:59,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279750 2024-12-09T08:17:59,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,755 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279750) roll requested 2024-12-09T08:17:59,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279755 2024-12-09T08:17:59,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279755 2024-12-09T08:17:59,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279755) roll requested 2024-12-09T08:17:59,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279760 2024-12-09T08:17:59,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279755 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279760 2024-12-09T08:17:59,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279760) roll requested 2024-12-09T08:17:59,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279768 2024-12-09T08:17:59,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279768 2024-12-09T08:17:59,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,773 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279768) roll requested 2024-12-09T08:17:59,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279773 2024-12-09T08:17:59,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279773 2024-12-09T08:17:59,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,779 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279773) roll requested 2024-12-09T08:17:59,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279779 2024-12-09T08:17:59,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279779 2024-12-09T08:17:59,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279779) roll requested 2024-12-09T08:17:59,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279785 2024-12-09T08:17:59,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279785 2024-12-09T08:17:59,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279785) roll requested 2024-12-09T08:17:59,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279791 2024-12-09T08:17:59,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279791 2024-12-09T08:17:59,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279791) roll requested 2024-12-09T08:17:59,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279800 2024-12-09T08:17:59,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279800 2024-12-09T08:17:59,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279800) roll requested 2024-12-09T08:17:59,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279808 2024-12-09T08:17:59,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279808 2024-12-09T08:17:59,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,815 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279808) roll requested 2024-12-09T08:17:59,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279815 2024-12-09T08:17:59,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279815 2024-12-09T08:17:59,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,823 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279815) roll requested 2024-12-09T08:17:59,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279824 2024-12-09T08:17:59,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279815 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279824 2024-12-09T08:17:59,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279824) roll requested 2024-12-09T08:17:59,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279831 2024-12-09T08:17:59,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279824 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279831 2024-12-09T08:17:59,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,838 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279831) roll requested 2024-12-09T08:17:59,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279839 2024-12-09T08:17:59,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279831 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279839 2024-12-09T08:17:59,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279839) roll requested 2024-12-09T08:17:59,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279848 2024-12-09T08:17:59,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279839 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279848 2024-12-09T08:17:59,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279848) roll requested 2024-12-09T08:17:59,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279859 2024-12-09T08:17:59,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279859 2024-12-09T08:17:59,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279859) roll requested 2024-12-09T08:17:59,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279868 2024-12-09T08:17:59,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279859 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279868 2024-12-09T08:17:59,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279868) roll requested 2024-12-09T08:17:59,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279876 2024-12-09T08:17:59,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279876 2024-12-09T08:17:59,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279876) roll requested 2024-12-09T08:17:59,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279883 2024-12-09T08:17:59,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279883 2024-12-09T08:17:59,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,889 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279883) roll requested 2024-12-09T08:17:59,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279889 2024-12-09T08:17:59,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279883 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279889 2024-12-09T08:17:59,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,898 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279889) roll requested 2024-12-09T08:17:59,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279898 2024-12-09T08:17:59,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279898 2024-12-09T08:17:59,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279898) roll requested 2024-12-09T08:17:59,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279908 2024-12-09T08:17:59,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279898 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279908 2024-12-09T08:17:59,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279908) roll requested 2024-12-09T08:17:59,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279916 2024-12-09T08:17:59,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279916 2024-12-09T08:17:59,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,923 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279916) roll requested 2024-12-09T08:17:59,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279924 2024-12-09T08:17:59,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279924 2024-12-09T08:17:59,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279924) roll requested 2024-12-09T08:17:59,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279933 2024-12-09T08:17:59,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279933 2024-12-09T08:17:59,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279933) roll requested 2024-12-09T08:17:59,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279941 2024-12-09T08:17:59,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279941 2024-12-09T08:17:59,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279941) roll requested 2024-12-09T08:17:59,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279947 2024-12-09T08:17:59,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279941 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279947 2024-12-09T08:17:59,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,954 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279947) roll requested 2024-12-09T08:17:59,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279955 2024-12-09T08:17:59,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279955 2024-12-09T08:17:59,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,963 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279955) roll requested 2024-12-09T08:17:59,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279964 2024-12-09T08:17:59,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279955 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279964 2024-12-09T08:17:59,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,974 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279964) roll requested 2024-12-09T08:17:59,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279974 2024-12-09T08:17:59,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279974 2024-12-09T08:17:59,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,984 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279974) roll requested 2024-12-09T08:17:59,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279984 2024-12-09T08:17:59,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279974 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279984 2024-12-09T08:17:59,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:17:59,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:17:59,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279984) roll requested 2024-12-09T08:17:59,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732279992 2024-12-09T08:17:59,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:17:59,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279984 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279992 2024-12-09T08:18:00,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732279992) roll requested 2024-12-09T08:18:00,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280000 2024-12-09T08:18:00,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732279992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280000 2024-12-09T08:18:00,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280000) roll requested 2024-12-09T08:18:00,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280007 2024-12-09T08:18:00,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280007 2024-12-09T08:18:00,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280007) roll requested 2024-12-09T08:18:00,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280014 2024-12-09T08:18:00,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280014 2024-12-09T08:18:00,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280014) roll requested 2024-12-09T08:18:00,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280023 2024-12-09T08:18:00,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280023 2024-12-09T08:18:00,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,031 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280023) roll requested 2024-12-09T08:18:00,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280031 2024-12-09T08:18:00,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280031 2024-12-09T08:18:00,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,039 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280031) roll requested 2024-12-09T08:18:00,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280040 2024-12-09T08:18:00,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280031 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280040 2024-12-09T08:18:00,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280040) roll requested 2024-12-09T08:18:00,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280048 2024-12-09T08:18:00,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280048 2024-12-09T08:18:00,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,057 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280048) roll requested 2024-12-09T08:18:00,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280057 2024-12-09T08:18:00,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280057 2024-12-09T08:18:00,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,065 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280057) roll requested 2024-12-09T08:18:00,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280066 2024-12-09T08:18:00,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280066 2024-12-09T08:18:00,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,075 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280066) roll requested 2024-12-09T08:18:00,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280075 2024-12-09T08:18:00,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280066 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280075 2024-12-09T08:18:00,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280075) roll requested 2024-12-09T08:18:00,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280084 2024-12-09T08:18:00,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280075 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280084 2024-12-09T08:18:00,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,094 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280084) roll requested 2024-12-09T08:18:00,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280094 2024-12-09T08:18:00,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280084 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280094 2024-12-09T08:18:00,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280094) roll requested 2024-12-09T08:18:00,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280103 2024-12-09T08:18:00,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280094 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280103 2024-12-09T08:18:00,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280103) roll requested 2024-12-09T08:18:00,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280108 2024-12-09T08:18:00,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280108 2024-12-09T08:18:00,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280108) roll requested 2024-12-09T08:18:00,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280113 2024-12-09T08:18:00,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280113 2024-12-09T08:18:00,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280113) roll requested 2024-12-09T08:18:00,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280120 2024-12-09T08:18:00,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280120 2024-12-09T08:18:00,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,126 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280120) roll requested 2024-12-09T08:18:00,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280127 2024-12-09T08:18:00,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280127 2024-12-09T08:18:00,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280127) roll requested 2024-12-09T08:18:00,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280134 2024-12-09T08:18:00,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280127 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280134 2024-12-09T08:18:00,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280134) roll requested 2024-12-09T08:18:00,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280142 2024-12-09T08:18:00,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280134 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280142 2024-12-09T08:18:00,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280142) roll requested 2024-12-09T08:18:00,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280151 2024-12-09T08:18:00,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280151 2024-12-09T08:18:00,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280151) roll requested 2024-12-09T08:18:00,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280160 2024-12-09T08:18:00,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280160 2024-12-09T08:18:00,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280160) roll requested 2024-12-09T08:18:00,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280169 2024-12-09T08:18:00,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280169 2024-12-09T08:18:00,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,178 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280169) roll requested 2024-12-09T08:18:00,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280179 2024-12-09T08:18:00,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280179 2024-12-09T08:18:00,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,215 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280179) roll requested 2024-12-09T08:18:00,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280216 2024-12-09T08:18:00,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280216 2024-12-09T08:18:00,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280216) roll requested 2024-12-09T08:18:00,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280226 2024-12-09T08:18:00,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280226 2024-12-09T08:18:00,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280226) roll requested 2024-12-09T08:18:00,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280231 2024-12-09T08:18:00,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280231 2024-12-09T08:18:00,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280231) roll requested 2024-12-09T08:18:00,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280237 2024-12-09T08:18:00,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280237 2024-12-09T08:18:00,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280237) roll requested 2024-12-09T08:18:00,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280245 2024-12-09T08:18:00,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280245 2024-12-09T08:18:00,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280245) roll requested 2024-12-09T08:18:00,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280251 2024-12-09T08:18:00,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280251 2024-12-09T08:18:00,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280251) roll requested 2024-12-09T08:18:00,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280257 2024-12-09T08:18:00,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280257 2024-12-09T08:18:00,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,265 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280257) roll requested 2024-12-09T08:18:00,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280266 2024-12-09T08:18:00,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280266 2024-12-09T08:18:00,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,274 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280266) roll requested 2024-12-09T08:18:00,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280274 2024-12-09T08:18:00,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280266 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280274 2024-12-09T08:18:00,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280274) roll requested 2024-12-09T08:18:00,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280282 2024-12-09T08:18:00,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280274 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280282 2024-12-09T08:18:00,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,290 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280282) roll requested 2024-12-09T08:18:00,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280290 2024-12-09T08:18:00,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280282 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280290 2024-12-09T08:18:00,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280290) roll requested 2024-12-09T08:18:00,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280299 2024-12-09T08:18:00,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280290 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280299 2024-12-09T08:18:00,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280299) roll requested 2024-12-09T08:18:00,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280308 2024-12-09T08:18:00,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280308 2024-12-09T08:18:00,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,316 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280308) roll requested 2024-12-09T08:18:00,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280317 2024-12-09T08:18:00,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280317 2024-12-09T08:18:00,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,325 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280317) roll requested 2024-12-09T08:18:00,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280325 2024-12-09T08:18:00,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280325 2024-12-09T08:18:00,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,333 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280325) roll requested 2024-12-09T08:18:00,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280333 2024-12-09T08:18:00,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280333 2024-12-09T08:18:00,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280333) roll requested 2024-12-09T08:18:00,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280340 2024-12-09T08:18:00,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280340 2024-12-09T08:18:00,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,349 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280340) roll requested 2024-12-09T08:18:00,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280349 2024-12-09T08:18:00,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280349 2024-12-09T08:18:00,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,359 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280349) roll requested 2024-12-09T08:18:00,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280359 2024-12-09T08:18:00,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280349 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280359 2024-12-09T08:18:00,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,367 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,367 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280359) roll requested 2024-12-09T08:18:00,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280367 2024-12-09T08:18:00,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280367 2024-12-09T08:18:00,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280367) roll requested 2024-12-09T08:18:00,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280376 2024-12-09T08:18:00,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280367 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280376 2024-12-09T08:18:00,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,385 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280376) roll requested 2024-12-09T08:18:00,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280386 2024-12-09T08:18:00,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280386 2024-12-09T08:18:00,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,395 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280386) roll requested 2024-12-09T08:18:00,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280395 2024-12-09T08:18:00,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280386 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280395 2024-12-09T08:18:00,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280395) roll requested 2024-12-09T08:18:00,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280405 2024-12-09T08:18:00,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280395 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280405 2024-12-09T08:18:00,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280405) roll requested 2024-12-09T08:18:00,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280414 2024-12-09T08:18:00,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280405 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280414 2024-12-09T08:18:00,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280414) roll requested 2024-12-09T08:18:00,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280420 2024-12-09T08:18:00,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280420 2024-12-09T08:18:00,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280420) roll requested 2024-12-09T08:18:00,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280427 2024-12-09T08:18:00,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280420 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280427 2024-12-09T08:18:00,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280427) roll requested 2024-12-09T08:18:00,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280432 2024-12-09T08:18:00,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280432 2024-12-09T08:18:00,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280432) roll requested 2024-12-09T08:18:00,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280437 2024-12-09T08:18:00,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280437 2024-12-09T08:18:00,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,443 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280437) roll requested 2024-12-09T08:18:00,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280444 2024-12-09T08:18:00,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280444 2024-12-09T08:18:00,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,451 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280444) roll requested 2024-12-09T08:18:00,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280451 2024-12-09T08:18:00,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280444 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280451 2024-12-09T08:18:00,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,458 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280451) roll requested 2024-12-09T08:18:00,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280458 2024-12-09T08:18:00,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280451 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280458 2024-12-09T08:18:00,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280458) roll requested 2024-12-09T08:18:00,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280467 2024-12-09T08:18:00,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280458 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280467 2024-12-09T08:18:00,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280467) roll requested 2024-12-09T08:18:00,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280474 2024-12-09T08:18:00,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280474 2024-12-09T08:18:00,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280474) roll requested 2024-12-09T08:18:00,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280484 2024-12-09T08:18:00,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280484 2024-12-09T08:18:00,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,493 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280484) roll requested 2024-12-09T08:18:00,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280494 2024-12-09T08:18:00,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280494 2024-12-09T08:18:00,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280494) roll requested 2024-12-09T08:18:00,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280504 2024-12-09T08:18:00,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280494 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280504 2024-12-09T08:18:00,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280504) roll requested 2024-12-09T08:18:00,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280515 2024-12-09T08:18:00,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280504 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280515 2024-12-09T08:18:00,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280515) roll requested 2024-12-09T08:18:00,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280521 2024-12-09T08:18:00,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280521 2024-12-09T08:18:00,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280521) roll requested 2024-12-09T08:18:00,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280527 2024-12-09T08:18:00,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280527 2024-12-09T08:18:00,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,532 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280527) roll requested 2024-12-09T08:18:00,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280533 2024-12-09T08:18:00,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280533 2024-12-09T08:18:00,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280533) roll requested 2024-12-09T08:18:00,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280539 2024-12-09T08:18:00,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280539 2024-12-09T08:18:00,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280539) roll requested 2024-12-09T08:18:00,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280548 2024-12-09T08:18:00,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280548 2024-12-09T08:18:00,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280548) roll requested 2024-12-09T08:18:00,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280559 2024-12-09T08:18:00,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280559 2024-12-09T08:18:00,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280559) roll requested 2024-12-09T08:18:00,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280565 2024-12-09T08:18:00,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280565 2024-12-09T08:18:00,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280565) roll requested 2024-12-09T08:18:00,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280571 2024-12-09T08:18:00,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280571 2024-12-09T08:18:00,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280571) roll requested 2024-12-09T08:18:00,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280576 2024-12-09T08:18:00,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280576 2024-12-09T08:18:00,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280576) roll requested 2024-12-09T08:18:00,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280584 2024-12-09T08:18:00,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280584 2024-12-09T08:18:00,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280584) roll requested 2024-12-09T08:18:00,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280589 2024-12-09T08:18:00,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280589 2024-12-09T08:18:00,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,594 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280589) roll requested 2024-12-09T08:18:00,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280594 2024-12-09T08:18:00,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280594 2024-12-09T08:18:00,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280594) roll requested 2024-12-09T08:18:00,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280600 2024-12-09T08:18:00,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280600 2024-12-09T08:18:00,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280600) roll requested 2024-12-09T08:18:00,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280605 2024-12-09T08:18:00,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280600 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280605 2024-12-09T08:18:00,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,610 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280605) roll requested 2024-12-09T08:18:00,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280610 2024-12-09T08:18:00,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280610 2024-12-09T08:18:00,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,615 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280610) roll requested 2024-12-09T08:18:00,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280615 2024-12-09T08:18:00,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280610 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280615 2024-12-09T08:18:00,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280615) roll requested 2024-12-09T08:18:00,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280621 2024-12-09T08:18:00,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280615 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280621 2024-12-09T08:18:00,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,626 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280621) roll requested 2024-12-09T08:18:00,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280626 2024-12-09T08:18:00,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280626 2024-12-09T08:18:00,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280626) roll requested 2024-12-09T08:18:00,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280631 2024-12-09T08:18:00,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280626 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280631 2024-12-09T08:18:00,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280631) roll requested 2024-12-09T08:18:00,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280637 2024-12-09T08:18:00,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280637 2024-12-09T08:18:00,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,643 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280637) roll requested 2024-12-09T08:18:00,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280643 2024-12-09T08:18:00,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280637 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280643 2024-12-09T08:18:00,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280643) roll requested 2024-12-09T08:18:00,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280648 2024-12-09T08:18:00,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280643 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280648 2024-12-09T08:18:00,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280648) roll requested 2024-12-09T08:18:00,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280653 2024-12-09T08:18:00,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280653 2024-12-09T08:18:00,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,658 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280653) roll requested 2024-12-09T08:18:00,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280658 2024-12-09T08:18:00,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280658 2024-12-09T08:18:00,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,663 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280658) roll requested 2024-12-09T08:18:00,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280663 2024-12-09T08:18:00,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280658 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280663 2024-12-09T08:18:00,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280663) roll requested 2024-12-09T08:18:00,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280668 2024-12-09T08:18:00,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280663 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280668 2024-12-09T08:18:00,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,673 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280668) roll requested 2024-12-09T08:18:00,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280673 2024-12-09T08:18:00,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280673 2024-12-09T08:18:00,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,678 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280673) roll requested 2024-12-09T08:18:00,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280678 2024-12-09T08:18:00,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280678 2024-12-09T08:18:00,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280678) roll requested 2024-12-09T08:18:00,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280684 2024-12-09T08:18:00,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280684 2024-12-09T08:18:00,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280684) roll requested 2024-12-09T08:18:00,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280689 2024-12-09T08:18:00,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280689 2024-12-09T08:18:00,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,693 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280689) roll requested 2024-12-09T08:18:00,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280694 2024-12-09T08:18:00,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280694 2024-12-09T08:18:00,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,699 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280694) roll requested 2024-12-09T08:18:00,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280700 2024-12-09T08:18:00,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280694 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280700 2024-12-09T08:18:00,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280700) roll requested 2024-12-09T08:18:00,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280704 2024-12-09T08:18:00,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280704 2024-12-09T08:18:00,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280704) roll requested 2024-12-09T08:18:00,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280709 2024-12-09T08:18:00,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280709 2024-12-09T08:18:00,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280709) roll requested 2024-12-09T08:18:00,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280714 2024-12-09T08:18:00,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280714 2024-12-09T08:18:00,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280714) roll requested 2024-12-09T08:18:00,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280719 2024-12-09T08:18:00,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280719 2024-12-09T08:18:00,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280719) roll requested 2024-12-09T08:18:00,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280724 2024-12-09T08:18:00,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280724 2024-12-09T08:18:00,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280724) roll requested 2024-12-09T08:18:00,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280730 2024-12-09T08:18:00,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280724 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280730 2024-12-09T08:18:00,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280730) roll requested 2024-12-09T08:18:00,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280734 2024-12-09T08:18:00,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280730 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280734 2024-12-09T08:18:00,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280734) roll requested 2024-12-09T08:18:00,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280739 2024-12-09T08:18:00,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280739 2024-12-09T08:18:00,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280739) roll requested 2024-12-09T08:18:00,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280745 2024-12-09T08:18:00,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280739 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280745 2024-12-09T08:18:00,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280745) roll requested 2024-12-09T08:18:00,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280752 2024-12-09T08:18:00,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280752 2024-12-09T08:18:00,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280752) roll requested 2024-12-09T08:18:00,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280760 2024-12-09T08:18:00,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280752 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280760 2024-12-09T08:18:00,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,767 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280760) roll requested 2024-12-09T08:18:00,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280768 2024-12-09T08:18:00,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280768 2024-12-09T08:18:00,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280768) roll requested 2024-12-09T08:18:00,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280775 2024-12-09T08:18:00,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280775 2024-12-09T08:18:00,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280775) roll requested 2024-12-09T08:18:00,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280784 2024-12-09T08:18:00,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280784 2024-12-09T08:18:00,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,792 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280784) roll requested 2024-12-09T08:18:00,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280792 2024-12-09T08:18:00,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280792 2024-12-09T08:18:00,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280792) roll requested 2024-12-09T08:18:00,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280803 2024-12-09T08:18:00,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280792 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280803 2024-12-09T08:18:00,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280803) roll requested 2024-12-09T08:18:00,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280813 2024-12-09T08:18:00,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280813 2024-12-09T08:18:00,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280813) roll requested 2024-12-09T08:18:00,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280821 2024-12-09T08:18:00,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280821 2024-12-09T08:18:00,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,826 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280821) roll requested 2024-12-09T08:18:00,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280826 2024-12-09T08:18:00,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280826 2024-12-09T08:18:00,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280826) roll requested 2024-12-09T08:18:00,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280831 2024-12-09T08:18:00,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280831 2024-12-09T08:18:00,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280831) roll requested 2024-12-09T08:18:00,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280836 2024-12-09T08:18:00,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280831 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280836 2024-12-09T08:18:00,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280836) roll requested 2024-12-09T08:18:00,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280841 2024-12-09T08:18:00,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280841 2024-12-09T08:18:00,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280841) roll requested 2024-12-09T08:18:00,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280845 2024-12-09T08:18:00,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280845 2024-12-09T08:18:00,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,851 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280845) roll requested 2024-12-09T08:18:00,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280852 2024-12-09T08:18:00,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280852 2024-12-09T08:18:00,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,858 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280852) roll requested 2024-12-09T08:18:00,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280858 2024-12-09T08:18:00,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280858 2024-12-09T08:18:00,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,866 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280858) roll requested 2024-12-09T08:18:00,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280866 2024-12-09T08:18:00,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280858 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280866 2024-12-09T08:18:00,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280866) roll requested 2024-12-09T08:18:00,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280873 2024-12-09T08:18:00,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280866 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280873 2024-12-09T08:18:00,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280873) roll requested 2024-12-09T08:18:00,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280881 2024-12-09T08:18:00,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280873 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280881 2024-12-09T08:18:00,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,889 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280881) roll requested 2024-12-09T08:18:00,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280890 2024-12-09T08:18:00,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280890 2024-12-09T08:18:00,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,898 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280890) roll requested 2024-12-09T08:18:00,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280898 2024-12-09T08:18:00,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280890 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280898 2024-12-09T08:18:00,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,909 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280898) roll requested 2024-12-09T08:18:00,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280909 2024-12-09T08:18:00,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280898 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280909 2024-12-09T08:18:00,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280909) roll requested 2024-12-09T08:18:00,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280917 2024-12-09T08:18:00,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280917 2024-12-09T08:18:00,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280917) roll requested 2024-12-09T08:18:00,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280925 2024-12-09T08:18:00,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280925 2024-12-09T08:18:00,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280925) roll requested 2024-12-09T08:18:00,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280933 2024-12-09T08:18:00,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280933 2024-12-09T08:18:00,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280933) roll requested 2024-12-09T08:18:00,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280941 2024-12-09T08:18:00,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280941 2024-12-09T08:18:00,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,950 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280941) roll requested 2024-12-09T08:18:00,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280950 2024-12-09T08:18:00,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280941 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280950 2024-12-09T08:18:00,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,959 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280950) roll requested 2024-12-09T08:18:00,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280960 2024-12-09T08:18:00,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280950 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280960 2024-12-09T08:18:00,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,969 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280960) roll requested 2024-12-09T08:18:00,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280969 2024-12-09T08:18:00,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280969 2024-12-09T08:18:00,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,975 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280969) roll requested 2024-12-09T08:18:00,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280975 2024-12-09T08:18:00,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280975 2024-12-09T08:18:00,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,984 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280975) roll requested 2024-12-09T08:18:00,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280985 2024-12-09T08:18:00,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280975 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280985 2024-12-09T08:18:00,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,993 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280985) roll requested 2024-12-09T08:18:00,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280993 2024-12-09T08:18:00,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:00,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280993 2024-12-09T08:18:00,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:00,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:00,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280993) roll requested 2024-12-09T08:18:00,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732280998 2024-12-09T08:18:01,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280998 2024-12-09T08:18:01,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732280998) roll requested 2024-12-09T08:18:01,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281005 2024-12-09T08:18:01,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732280998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281005 2024-12-09T08:18:01,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281005) roll requested 2024-12-09T08:18:01,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281011 2024-12-09T08:18:01,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281005 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281011 2024-12-09T08:18:01,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281011) roll requested 2024-12-09T08:18:01,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281016 2024-12-09T08:18:01,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281016 2024-12-09T08:18:01,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281016) roll requested 2024-12-09T08:18:01,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281021 2024-12-09T08:18:01,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281021 2024-12-09T08:18:01,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281021) roll requested 2024-12-09T08:18:01,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281026 2024-12-09T08:18:01,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281026 2024-12-09T08:18:01,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281026) roll requested 2024-12-09T08:18:01,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281032 2024-12-09T08:18:01,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281032 2024-12-09T08:18:01,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,038 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281032) roll requested 2024-12-09T08:18:01,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281039 2024-12-09T08:18:01,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281039 2024-12-09T08:18:01,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281039) roll requested 2024-12-09T08:18:01,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281049 2024-12-09T08:18:01,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281039 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281049 2024-12-09T08:18:01,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281049) roll requested 2024-12-09T08:18:01,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281059 2024-12-09T08:18:01,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281059 2024-12-09T08:18:01,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281059) roll requested 2024-12-09T08:18:01,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281064 2024-12-09T08:18:01,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281059 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281064 2024-12-09T08:18:01,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281064) roll requested 2024-12-09T08:18:01,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281070 2024-12-09T08:18:01,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281070 2024-12-09T08:18:01,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,074 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281070) roll requested 2024-12-09T08:18:01,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281075 2024-12-09T08:18:01,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281075 2024-12-09T08:18:01,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,079 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281075) roll requested 2024-12-09T08:18:01,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281079 2024-12-09T08:18:01,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281075 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281079 2024-12-09T08:18:01,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281079) roll requested 2024-12-09T08:18:01,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281083 2024-12-09T08:18:01,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281079 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281083 2024-12-09T08:18:01,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281083) roll requested 2024-12-09T08:18:01,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281090 2024-12-09T08:18:01,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281090 2024-12-09T08:18:01,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,097 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281090) roll requested 2024-12-09T08:18:01,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281098 2024-12-09T08:18:01,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281098 2024-12-09T08:18:01,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281098) roll requested 2024-12-09T08:18:01,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281107 2024-12-09T08:18:01,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281098 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281107 2024-12-09T08:18:01,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,118 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281107) roll requested 2024-12-09T08:18:01,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281119 2024-12-09T08:18:01,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281119 2024-12-09T08:18:01,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281119) roll requested 2024-12-09T08:18:01,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281124 2024-12-09T08:18:01,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281119 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281124 2024-12-09T08:18:01,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281124) roll requested 2024-12-09T08:18:01,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281129 2024-12-09T08:18:01,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281129 2024-12-09T08:18:01,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281129) roll requested 2024-12-09T08:18:01,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281137 2024-12-09T08:18:01,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281137 2024-12-09T08:18:01,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,147 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281137) roll requested 2024-12-09T08:18:01,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281148 2024-12-09T08:18:01,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281148 2024-12-09T08:18:01,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281148) roll requested 2024-12-09T08:18:01,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281153 2024-12-09T08:18:01,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281153 2024-12-09T08:18:01,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281153) roll requested 2024-12-09T08:18:01,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281161 2024-12-09T08:18:01,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281161 2024-12-09T08:18:01,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281161) roll requested 2024-12-09T08:18:01,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281166 2024-12-09T08:18:01,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281166 2024-12-09T08:18:01,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281166) roll requested 2024-12-09T08:18:01,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281171 2024-12-09T08:18:01,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281166 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281171 2024-12-09T08:18:01,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,177 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281171) roll requested 2024-12-09T08:18:01,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281177 2024-12-09T08:18:01,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281177 2024-12-09T08:18:01,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,182 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281177) roll requested 2024-12-09T08:18:01,182 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281182 2024-12-09T08:18:01,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281182 2024-12-09T08:18:01,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281182) roll requested 2024-12-09T08:18:01,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281189 2024-12-09T08:18:01,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281182 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281189 2024-12-09T08:18:01,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281189) roll requested 2024-12-09T08:18:01,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281197 2024-12-09T08:18:01,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281197 2024-12-09T08:18:01,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281197) roll requested 2024-12-09T08:18:01,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281202 2024-12-09T08:18:01,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281202 2024-12-09T08:18:01,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,207 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281202) roll requested 2024-12-09T08:18:01,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281207 2024-12-09T08:18:01,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281207 2024-12-09T08:18:01,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,213 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281207) roll requested 2024-12-09T08:18:01,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281213 2024-12-09T08:18:01,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281207 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281213 2024-12-09T08:18:01,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281213) roll requested 2024-12-09T08:18:01,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281218 2024-12-09T08:18:01,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281218 2024-12-09T08:18:01,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281218) roll requested 2024-12-09T08:18:01,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281225 2024-12-09T08:18:01,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281218 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281225 2024-12-09T08:18:01,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281225) roll requested 2024-12-09T08:18:01,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281229 2024-12-09T08:18:01,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281229 2024-12-09T08:18:01,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,234 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281229) roll requested 2024-12-09T08:18:01,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281234 2024-12-09T08:18:01,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281234 2024-12-09T08:18:01,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281234) roll requested 2024-12-09T08:18:01,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281239 2024-12-09T08:18:01,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281239 2024-12-09T08:18:01,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281239) roll requested 2024-12-09T08:18:01,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281245 2024-12-09T08:18:01,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281239 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281245 2024-12-09T08:18:01,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281245) roll requested 2024-12-09T08:18:01,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281252 2024-12-09T08:18:01,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281252 2024-12-09T08:18:01,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281252) roll requested 2024-12-09T08:18:01,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281260 2024-12-09T08:18:01,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281260 2024-12-09T08:18:01,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281260) roll requested 2024-12-09T08:18:01,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281268 2024-12-09T08:18:01,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281260 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281268 2024-12-09T08:18:01,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281268) roll requested 2024-12-09T08:18:01,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281277 2024-12-09T08:18:01,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281277 2024-12-09T08:18:01,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281277) roll requested 2024-12-09T08:18:01,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281286 2024-12-09T08:18:01,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281286 2024-12-09T08:18:01,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281286) roll requested 2024-12-09T08:18:01,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281294 2024-12-09T08:18:01,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281294 2024-12-09T08:18:01,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281294) roll requested 2024-12-09T08:18:01,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281301 2024-12-09T08:18:01,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281301 2024-12-09T08:18:01,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281301) roll requested 2024-12-09T08:18:01,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281308 2024-12-09T08:18:01,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281308 2024-12-09T08:18:01,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,316 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281308) roll requested 2024-12-09T08:18:01,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281316 2024-12-09T08:18:01,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281316 2024-12-09T08:18:01,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281316) roll requested 2024-12-09T08:18:01,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281324 2024-12-09T08:18:01,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281316 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281324 2024-12-09T08:18:01,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281324) roll requested 2024-12-09T08:18:01,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281333 2024-12-09T08:18:01,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281333 2024-12-09T08:18:01,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281333) roll requested 2024-12-09T08:18:01,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281339 2024-12-09T08:18:01,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281339 2024-12-09T08:18:01,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281339) roll requested 2024-12-09T08:18:01,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281347 2024-12-09T08:18:01,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281347 2024-12-09T08:18:01,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,355 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281347) roll requested 2024-12-09T08:18:01,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281355 2024-12-09T08:18:01,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281355 2024-12-09T08:18:01,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281355) roll requested 2024-12-09T08:18:01,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281363 2024-12-09T08:18:01,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281363 2024-12-09T08:18:01,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281363) roll requested 2024-12-09T08:18:01,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281372 2024-12-09T08:18:01,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281372 2024-12-09T08:18:01,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,380 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281372) roll requested 2024-12-09T08:18:01,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281381 2024-12-09T08:18:01,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281381 2024-12-09T08:18:01,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281381) roll requested 2024-12-09T08:18:01,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281387 2024-12-09T08:18:01,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281381 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281387 2024-12-09T08:18:01,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281387) roll requested 2024-12-09T08:18:01,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281395 2024-12-09T08:18:01,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281395 2024-12-09T08:18:01,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,403 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281395) roll requested 2024-12-09T08:18:01,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281404 2024-12-09T08:18:01,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281395 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281404 2024-12-09T08:18:01,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281404) roll requested 2024-12-09T08:18:01,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281415 2024-12-09T08:18:01,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281415 2024-12-09T08:18:01,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,423 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281415) roll requested 2024-12-09T08:18:01,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281423 2024-12-09T08:18:01,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281423 2024-12-09T08:18:01,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,430 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281423) roll requested 2024-12-09T08:18:01,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281430 2024-12-09T08:18:01,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281423 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281430 2024-12-09T08:18:01,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,438 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281430) roll requested 2024-12-09T08:18:01,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281438 2024-12-09T08:18:01,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281438 2024-12-09T08:18:01,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281438) roll requested 2024-12-09T08:18:01,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281447 2024-12-09T08:18:01,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281438 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281447 2024-12-09T08:18:01,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,455 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281447) roll requested 2024-12-09T08:18:01,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281456 2024-12-09T08:18:01,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281456 2024-12-09T08:18:01,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281456) roll requested 2024-12-09T08:18:01,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281465 2024-12-09T08:18:01,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281465 2024-12-09T08:18:01,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281465) roll requested 2024-12-09T08:18:01,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281475 2024-12-09T08:18:01,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281475 2024-12-09T08:18:01,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281475) roll requested 2024-12-09T08:18:01,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281483 2024-12-09T08:18:01,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281475 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281483 2024-12-09T08:18:01,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281483) roll requested 2024-12-09T08:18:01,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281490 2024-12-09T08:18:01,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281490 2024-12-09T08:18:01,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,498 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281490) roll requested 2024-12-09T08:18:01,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281499 2024-12-09T08:18:01,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281499 2024-12-09T08:18:01,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281499) roll requested 2024-12-09T08:18:01,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281507 2024-12-09T08:18:01,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281499 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281507 2024-12-09T08:18:01,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281507) roll requested 2024-12-09T08:18:01,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281515 2024-12-09T08:18:01,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281515 2024-12-09T08:18:01,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281515) roll requested 2024-12-09T08:18:01,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281523 2024-12-09T08:18:01,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281523 2024-12-09T08:18:01,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281523) roll requested 2024-12-09T08:18:01,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281531 2024-12-09T08:18:01,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281523 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281531 2024-12-09T08:18:01,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281531) roll requested 2024-12-09T08:18:01,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281541 2024-12-09T08:18:01,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281541 2024-12-09T08:18:01,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281541) roll requested 2024-12-09T08:18:01,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281548 2024-12-09T08:18:01,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281548 2024-12-09T08:18:01,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281548) roll requested 2024-12-09T08:18:01,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281555 2024-12-09T08:18:01,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281555 2024-12-09T08:18:01,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281555) roll requested 2024-12-09T08:18:01,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281561 2024-12-09T08:18:01,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281561 2024-12-09T08:18:01,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,567 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281561) roll requested 2024-12-09T08:18:01,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281567 2024-12-09T08:18:01,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281567 2024-12-09T08:18:01,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,573 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281567) roll requested 2024-12-09T08:18:01,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281573 2024-12-09T08:18:01,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281567 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281573 2024-12-09T08:18:01,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281573) roll requested 2024-12-09T08:18:01,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281580 2024-12-09T08:18:01,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281573 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281580 2024-12-09T08:18:01,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281580) roll requested 2024-12-09T08:18:01,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281587 2024-12-09T08:18:01,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281587 2024-12-09T08:18:01,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,594 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281587) roll requested 2024-12-09T08:18:01,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281595 2024-12-09T08:18:01,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281595 2024-12-09T08:18:01,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281595) roll requested 2024-12-09T08:18:01,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281601 2024-12-09T08:18:01,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281601 2024-12-09T08:18:01,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,610 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281601) roll requested 2024-12-09T08:18:01,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281610 2024-12-09T08:18:01,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281610 2024-12-09T08:18:01,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,619 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281610) roll requested 2024-12-09T08:18:01,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281619 2024-12-09T08:18:01,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281610 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281619 2024-12-09T08:18:01,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281619) roll requested 2024-12-09T08:18:01,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281625 2024-12-09T08:18:01,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281625 2024-12-09T08:18:01,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281625) roll requested 2024-12-09T08:18:01,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281632 2024-12-09T08:18:01,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281632 2024-12-09T08:18:01,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,641 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281632) roll requested 2024-12-09T08:18:01,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281641 2024-12-09T08:18:01,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281641 2024-12-09T08:18:01,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281641) roll requested 2024-12-09T08:18:01,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281649 2024-12-09T08:18:01,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281649 2024-12-09T08:18:01,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,655 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281649) roll requested 2024-12-09T08:18:01,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281655 2024-12-09T08:18:01,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281655 2024-12-09T08:18:01,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,661 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281655) roll requested 2024-12-09T08:18:01,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281661 2024-12-09T08:18:01,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281655 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281661 2024-12-09T08:18:01,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281661) roll requested 2024-12-09T08:18:01,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281669 2024-12-09T08:18:01,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281669 2024-12-09T08:18:01,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281669) roll requested 2024-12-09T08:18:01,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281678 2024-12-09T08:18:01,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281678 2024-12-09T08:18:01,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281678) roll requested 2024-12-09T08:18:01,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281684 2024-12-09T08:18:01,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281684 2024-12-09T08:18:01,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281684) roll requested 2024-12-09T08:18:01,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281690 2024-12-09T08:18:01,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281690 2024-12-09T08:18:01,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281690) roll requested 2024-12-09T08:18:01,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281697 2024-12-09T08:18:01,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281697 2024-12-09T08:18:01,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281697) roll requested 2024-12-09T08:18:01,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281705 2024-12-09T08:18:01,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281705 2024-12-09T08:18:01,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281705) roll requested 2024-12-09T08:18:01,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281712 2024-12-09T08:18:01,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281712 2024-12-09T08:18:01,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281712) roll requested 2024-12-09T08:18:01,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281718 2024-12-09T08:18:01,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281718 2024-12-09T08:18:01,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281718) roll requested 2024-12-09T08:18:01,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281725 2024-12-09T08:18:01,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281718 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281725 2024-12-09T08:18:01,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281725) roll requested 2024-12-09T08:18:01,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281733 2024-12-09T08:18:01,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281733 2024-12-09T08:18:01,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281733) roll requested 2024-12-09T08:18:01,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281743 2024-12-09T08:18:01,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281743 2024-12-09T08:18:01,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281743) roll requested 2024-12-09T08:18:01,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281752 2024-12-09T08:18:01,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281743 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281752 2024-12-09T08:18:01,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281752) roll requested 2024-12-09T08:18:01,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281761 2024-12-09T08:18:01,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281752 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281761 2024-12-09T08:18:01,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281761) roll requested 2024-12-09T08:18:01,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281770 2024-12-09T08:18:01,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281770 2024-12-09T08:18:01,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281770) roll requested 2024-12-09T08:18:01,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281778 2024-12-09T08:18:01,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281770 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281778 2024-12-09T08:18:01,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281778) roll requested 2024-12-09T08:18:01,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281785 2024-12-09T08:18:01,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281785 2024-12-09T08:18:01,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,793 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281785) roll requested 2024-12-09T08:18:01,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281793 2024-12-09T08:18:01,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281793 2024-12-09T08:18:01,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,802 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281793) roll requested 2024-12-09T08:18:01,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281802 2024-12-09T08:18:01,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281802 2024-12-09T08:18:01,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281802) roll requested 2024-12-09T08:18:01,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281812 2024-12-09T08:18:01,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281802 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281812 2024-12-09T08:18:01,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281812) roll requested 2024-12-09T08:18:01,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281823 2024-12-09T08:18:01,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281823 2024-12-09T08:18:01,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281823) roll requested 2024-12-09T08:18:01,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281831 2024-12-09T08:18:01,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281823 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281831 2024-12-09T08:18:01,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281831) roll requested 2024-12-09T08:18:01,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281840 2024-12-09T08:18:01,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281831 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281840 2024-12-09T08:18:01,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281840) roll requested 2024-12-09T08:18:01,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281850 2024-12-09T08:18:01,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281850 2024-12-09T08:18:01,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,858 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281850) roll requested 2024-12-09T08:18:01,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281858 2024-12-09T08:18:01,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281858 2024-12-09T08:18:01,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281858) roll requested 2024-12-09T08:18:01,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281867 2024-12-09T08:18:01,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281858 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281867 2024-12-09T08:18:01,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281867) roll requested 2024-12-09T08:18:01,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281876 2024-12-09T08:18:01,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281876 2024-12-09T08:18:01,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281876) roll requested 2024-12-09T08:18:01,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281883 2024-12-09T08:18:01,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281883 2024-12-09T08:18:01,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281883) roll requested 2024-12-09T08:18:01,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281890 2024-12-09T08:18:01,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281883 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281890 2024-12-09T08:18:01,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,898 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281890) roll requested 2024-12-09T08:18:01,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281899 2024-12-09T08:18:01,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281890 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281899 2024-12-09T08:18:01,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281899) roll requested 2024-12-09T08:18:01,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281911 2024-12-09T08:18:01,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281899 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281911 2024-12-09T08:18:01,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281911) roll requested 2024-12-09T08:18:01,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281920 2024-12-09T08:18:01,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281920 2024-12-09T08:18:01,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,930 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281920) roll requested 2024-12-09T08:18:01,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281930 2024-12-09T08:18:01,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281930 2024-12-09T08:18:01,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281930) roll requested 2024-12-09T08:18:01,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281940 2024-12-09T08:18:01,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281930 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281940 2024-12-09T08:18:01,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,949 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281940) roll requested 2024-12-09T08:18:01,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281949 2024-12-09T08:18:01,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281949 2024-12-09T08:18:01,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281949) roll requested 2024-12-09T08:18:01,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281958 2024-12-09T08:18:01,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281958 2024-12-09T08:18:01,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281958) roll requested 2024-12-09T08:18:01,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281967 2024-12-09T08:18:01,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281967 2024-12-09T08:18:01,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,974 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281967) roll requested 2024-12-09T08:18:01,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281974 2024-12-09T08:18:01,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281967 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281974 2024-12-09T08:18:01,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281974) roll requested 2024-12-09T08:18:01,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281982 2024-12-09T08:18:01,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281974 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281982 2024-12-09T08:18:01,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,990 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281982) roll requested 2024-12-09T08:18:01,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281990 2024-12-09T08:18:01,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:01,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281990 2024-12-09T08:18:01,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:01,999 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:01,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281990) roll requested 2024-12-09T08:18:01,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732281999 2024-12-09T08:18:02,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281990 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281999 2024-12-09T08:18:02,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732281999) roll requested 2024-12-09T08:18:02,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282007 2024-12-09T08:18:02,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732281999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282007 2024-12-09T08:18:02,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282007) roll requested 2024-12-09T08:18:02,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282015 2024-12-09T08:18:02,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282015 2024-12-09T08:18:02,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282015) roll requested 2024-12-09T08:18:02,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282021 2024-12-09T08:18:02,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282015 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282021 2024-12-09T08:18:02,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282021) roll requested 2024-12-09T08:18:02,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282028 2024-12-09T08:18:02,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282028 2024-12-09T08:18:02,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,037 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282028) roll requested 2024-12-09T08:18:02,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282037 2024-12-09T08:18:02,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282028 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282037 2024-12-09T08:18:02,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,045 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282037) roll requested 2024-12-09T08:18:02,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282045 2024-12-09T08:18:02,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282045 2024-12-09T08:18:02,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,050 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282045) roll requested 2024-12-09T08:18:02,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282050 2024-12-09T08:18:02,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282045 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282050 2024-12-09T08:18:02,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282050) roll requested 2024-12-09T08:18:02,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282055 2024-12-09T08:18:02,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282050 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282055 2024-12-09T08:18:02,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282055) roll requested 2024-12-09T08:18:02,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282062 2024-12-09T08:18:02,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282062 2024-12-09T08:18:02,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282062) roll requested 2024-12-09T08:18:02,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282068 2024-12-09T08:18:02,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282068 2024-12-09T08:18:02,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,075 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282068) roll requested 2024-12-09T08:18:02,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282075 2024-12-09T08:18:02,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282075 2024-12-09T08:18:02,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282075) roll requested 2024-12-09T08:18:02,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282082 2024-12-09T08:18:02,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282075 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282082 2024-12-09T08:18:02,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282082) roll requested 2024-12-09T08:18:02,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282089 2024-12-09T08:18:02,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282089 2024-12-09T08:18:02,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282089) roll requested 2024-12-09T08:18:02,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282096 2024-12-09T08:18:02,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282096 2024-12-09T08:18:02,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282096) roll requested 2024-12-09T08:18:02,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282103 2024-12-09T08:18:02,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282103 2024-12-09T08:18:02,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,111 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282103) roll requested 2024-12-09T08:18:02,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282111 2024-12-09T08:18:02,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282111 2024-12-09T08:18:02,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,118 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282111) roll requested 2024-12-09T08:18:02,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282119 2024-12-09T08:18:02,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282111 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282119 2024-12-09T08:18:02,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,125 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282119) roll requested 2024-12-09T08:18:02,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282126 2024-12-09T08:18:02,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282119 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282126 2024-12-09T08:18:02,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282126) roll requested 2024-12-09T08:18:02,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282134 2024-12-09T08:18:02,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282126 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282134 2024-12-09T08:18:02,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282134) roll requested 2024-12-09T08:18:02,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282143 2024-12-09T08:18:02,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282134 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282143 2024-12-09T08:18:02,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282143) roll requested 2024-12-09T08:18:02,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282151 2024-12-09T08:18:02,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282151 2024-12-09T08:18:02,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282151) roll requested 2024-12-09T08:18:02,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282157 2024-12-09T08:18:02,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282157 2024-12-09T08:18:02,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282157) roll requested 2024-12-09T08:18:02,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282164 2024-12-09T08:18:02,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282164 2024-12-09T08:18:02,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282164) roll requested 2024-12-09T08:18:02,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282172 2024-12-09T08:18:02,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282164 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282172 2024-12-09T08:18:02,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,178 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282172) roll requested 2024-12-09T08:18:02,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282178 2024-12-09T08:18:02,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282178 2024-12-09T08:18:02,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282178) roll requested 2024-12-09T08:18:02,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282184 2024-12-09T08:18:02,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282178 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282184 2024-12-09T08:18:02,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,191 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282184) roll requested 2024-12-09T08:18:02,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282191 2024-12-09T08:18:02,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282191 2024-12-09T08:18:02,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,198 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282191) roll requested 2024-12-09T08:18:02,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282198 2024-12-09T08:18:02,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282191 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282198 2024-12-09T08:18:02,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282198) roll requested 2024-12-09T08:18:02,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282205 2024-12-09T08:18:02,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282198 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282205 2024-12-09T08:18:02,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282205) roll requested 2024-12-09T08:18:02,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282213 2024-12-09T08:18:02,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282213 2024-12-09T08:18:02,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,221 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282213) roll requested 2024-12-09T08:18:02,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282222 2024-12-09T08:18:02,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282222 2024-12-09T08:18:02,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,228 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282222) roll requested 2024-12-09T08:18:02,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282228 2024-12-09T08:18:02,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282228 2024-12-09T08:18:02,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282228) roll requested 2024-12-09T08:18:02,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282233 2024-12-09T08:18:02,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282233 2024-12-09T08:18:02,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282233) roll requested 2024-12-09T08:18:02,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282239 2024-12-09T08:18:02,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282239 2024-12-09T08:18:02,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,247 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282239) roll requested 2024-12-09T08:18:02,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282248 2024-12-09T08:18:02,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282239 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282248 2024-12-09T08:18:02,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282248) roll requested 2024-12-09T08:18:02,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282256 2024-12-09T08:18:02,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282256 2024-12-09T08:18:02,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282256) roll requested 2024-12-09T08:18:02,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282264 2024-12-09T08:18:02,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282264 2024-12-09T08:18:02,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282264) roll requested 2024-12-09T08:18:02,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282272 2024-12-09T08:18:02,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282264 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282272 2024-12-09T08:18:02,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282272) roll requested 2024-12-09T08:18:02,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282280 2024-12-09T08:18:02,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282272 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282280 2024-12-09T08:18:02,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282280) roll requested 2024-12-09T08:18:02,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282287 2024-12-09T08:18:02,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282287 2024-12-09T08:18:02,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282287) roll requested 2024-12-09T08:18:02,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282295 2024-12-09T08:18:02,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282295 2024-12-09T08:18:02,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282295) roll requested 2024-12-09T08:18:02,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282301 2024-12-09T08:18:02,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282295 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282301 2024-12-09T08:18:02,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282301) roll requested 2024-12-09T08:18:02,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282309 2024-12-09T08:18:02,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282309 2024-12-09T08:18:02,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282309) roll requested 2024-12-09T08:18:02,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282317 2024-12-09T08:18:02,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282309 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282317 2024-12-09T08:18:02,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,325 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282317) roll requested 2024-12-09T08:18:02,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282326 2024-12-09T08:18:02,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282326 2024-12-09T08:18:02,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,333 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282326) roll requested 2024-12-09T08:18:02,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282334 2024-12-09T08:18:02,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282326 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282334 2024-12-09T08:18:02,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,342 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282334) roll requested 2024-12-09T08:18:02,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282342 2024-12-09T08:18:02,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282342 2024-12-09T08:18:02,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,351 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282342) roll requested 2024-12-09T08:18:02,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282351 2024-12-09T08:18:02,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282342 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282351 2024-12-09T08:18:02,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,357 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282351) roll requested 2024-12-09T08:18:02,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282357 2024-12-09T08:18:02,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282351 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282357 2024-12-09T08:18:02,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,365 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282357) roll requested 2024-12-09T08:18:02,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282366 2024-12-09T08:18:02,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282357 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282366 2024-12-09T08:18:02,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282366) roll requested 2024-12-09T08:18:02,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282374 2024-12-09T08:18:02,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282366 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282374 2024-12-09T08:18:02,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282374) roll requested 2024-12-09T08:18:02,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282383 2024-12-09T08:18:02,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282383 2024-12-09T08:18:02,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,390 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282383) roll requested 2024-12-09T08:18:02,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282391 2024-12-09T08:18:02,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282391 2024-12-09T08:18:02,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,398 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282391) roll requested 2024-12-09T08:18:02,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282398 2024-12-09T08:18:02,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282391 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282398 2024-12-09T08:18:02,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282398) roll requested 2024-12-09T08:18:02,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282406 2024-12-09T08:18:02,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282398 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282406 2024-12-09T08:18:02,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282406) roll requested 2024-12-09T08:18:02,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282413 2024-12-09T08:18:02,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282413 2024-12-09T08:18:02,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282413) roll requested 2024-12-09T08:18:02,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282420 2024-12-09T08:18:02,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282420 2024-12-09T08:18:02,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282420) roll requested 2024-12-09T08:18:02,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282428 2024-12-09T08:18:02,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282420 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282428 2024-12-09T08:18:02,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282428) roll requested 2024-12-09T08:18:02,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282436 2024-12-09T08:18:02,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282436 2024-12-09T08:18:02,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282436) roll requested 2024-12-09T08:18:02,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282446 2024-12-09T08:18:02,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282436 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282446 2024-12-09T08:18:02,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282446) roll requested 2024-12-09T08:18:02,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282453 2024-12-09T08:18:02,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282446 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282453 2024-12-09T08:18:02,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,458 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282453) roll requested 2024-12-09T08:18:02,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282458 2024-12-09T08:18:02,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282458 2024-12-09T08:18:02,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,463 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282458) roll requested 2024-12-09T08:18:02,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282463 2024-12-09T08:18:02,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282458 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282463 2024-12-09T08:18:02,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,468 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282463) roll requested 2024-12-09T08:18:02,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282468 2024-12-09T08:18:02,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282463 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282468 2024-12-09T08:18:02,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282468) roll requested 2024-12-09T08:18:02,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282473 2024-12-09T08:18:02,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282468 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282473 2024-12-09T08:18:02,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,479 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282473) roll requested 2024-12-09T08:18:02,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282479 2024-12-09T08:18:02,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282479 2024-12-09T08:18:02,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282479) roll requested 2024-12-09T08:18:02,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282486 2024-12-09T08:18:02,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282479 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282486 2024-12-09T08:18:02,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,493 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282486) roll requested 2024-12-09T08:18:02,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282493 2024-12-09T08:18:02,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282486 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282493 2024-12-09T08:18:02,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,499 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282493) roll requested 2024-12-09T08:18:02,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282499 2024-12-09T08:18:02,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282499 2024-12-09T08:18:02,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,505 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282499) roll requested 2024-12-09T08:18:02,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282505 2024-12-09T08:18:02,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282499 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282505 2024-12-09T08:18:02,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,511 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282505) roll requested 2024-12-09T08:18:02,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282511 2024-12-09T08:18:02,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282511 2024-12-09T08:18:02,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282511) roll requested 2024-12-09T08:18:02,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282519 2024-12-09T08:18:02,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282511 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282519 2024-12-09T08:18:02,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282519) roll requested 2024-12-09T08:18:02,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282526 2024-12-09T08:18:02,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282526 2024-12-09T08:18:02,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,534 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282526) roll requested 2024-12-09T08:18:02,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282534 2024-12-09T08:18:02,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282534 2024-12-09T08:18:02,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,542 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282534) roll requested 2024-12-09T08:18:02,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282542 2024-12-09T08:18:02,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282534 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282542 2024-12-09T08:18:02,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,551 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282542) roll requested 2024-12-09T08:18:02,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282551 2024-12-09T08:18:02,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282542 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282551 2024-12-09T08:18:02,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282551) roll requested 2024-12-09T08:18:02,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282561 2024-12-09T08:18:02,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282551 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282561 2024-12-09T08:18:02,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282561) roll requested 2024-12-09T08:18:02,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282570 2024-12-09T08:18:02,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282570 2024-12-09T08:18:02,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282570) roll requested 2024-12-09T08:18:02,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282576 2024-12-09T08:18:02,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282570 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282576 2024-12-09T08:18:02,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282576) roll requested 2024-12-09T08:18:02,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282584 2024-12-09T08:18:02,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282584 2024-12-09T08:18:02,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282584) roll requested 2024-12-09T08:18:02,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282589 2024-12-09T08:18:02,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282589 2024-12-09T08:18:02,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282589) roll requested 2024-12-09T08:18:02,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282595 2024-12-09T08:18:02,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282595 2024-12-09T08:18:02,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282595) roll requested 2024-12-09T08:18:02,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282603 2024-12-09T08:18:02,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282603 2024-12-09T08:18:02,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282603) roll requested 2024-12-09T08:18:02,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282612 2024-12-09T08:18:02,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282603 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282612 2024-12-09T08:18:02,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,619 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282612) roll requested 2024-12-09T08:18:02,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282619 2024-12-09T08:18:02,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282619 2024-12-09T08:18:02,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282619) roll requested 2024-12-09T08:18:02,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282628 2024-12-09T08:18:02,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282628 2024-12-09T08:18:02,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,635 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282628) roll requested 2024-12-09T08:18:02,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282635 2024-12-09T08:18:02,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282635 2024-12-09T08:18:02,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282635) roll requested 2024-12-09T08:18:02,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282642 2024-12-09T08:18:02,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282635 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282642 2024-12-09T08:18:02,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282642) roll requested 2024-12-09T08:18:02,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282651 2024-12-09T08:18:02,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282642 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282651 2024-12-09T08:18:02,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,661 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282651) roll requested 2024-12-09T08:18:02,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282661 2024-12-09T08:18:02,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282661 2024-12-09T08:18:02,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282661) roll requested 2024-12-09T08:18:02,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282670 2024-12-09T08:18:02,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282670 2024-12-09T08:18:02,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,675 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282670) roll requested 2024-12-09T08:18:02,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282675 2024-12-09T08:18:02,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282670 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282675 2024-12-09T08:18:02,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282675) roll requested 2024-12-09T08:18:02,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282681 2024-12-09T08:18:02,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282675 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282681 2024-12-09T08:18:02,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,687 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282681) roll requested 2024-12-09T08:18:02,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282687 2024-12-09T08:18:02,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282687 2024-12-09T08:18:02,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282687) roll requested 2024-12-09T08:18:02,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282696 2024-12-09T08:18:02,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282687 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282696 2024-12-09T08:18:02,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282696) roll requested 2024-12-09T08:18:02,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282702 2024-12-09T08:18:02,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282696 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282702 2024-12-09T08:18:02,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282702) roll requested 2024-12-09T08:18:02,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282708 2024-12-09T08:18:02,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282708 2024-12-09T08:18:02,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,715 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282708) roll requested 2024-12-09T08:18:02,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282715 2024-12-09T08:18:02,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282715 2024-12-09T08:18:02,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,723 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282715) roll requested 2024-12-09T08:18:02,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282723 2024-12-09T08:18:02,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282723 2024-12-09T08:18:02,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,731 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282723) roll requested 2024-12-09T08:18:02,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282732 2024-12-09T08:18:02,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282723 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282732 2024-12-09T08:18:02,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282732) roll requested 2024-12-09T08:18:02,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282739 2024-12-09T08:18:02,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282739 2024-12-09T08:18:02,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,746 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282739) roll requested 2024-12-09T08:18:02,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282746 2024-12-09T08:18:02,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282739 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282746 2024-12-09T08:18:02,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,754 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282746) roll requested 2024-12-09T08:18:02,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282755 2024-12-09T08:18:02,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282755 2024-12-09T08:18:02,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282755) roll requested 2024-12-09T08:18:02,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282763 2024-12-09T08:18:02,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282755 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282763 2024-12-09T08:18:02,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,771 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282763) roll requested 2024-12-09T08:18:02,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282771 2024-12-09T08:18:02,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282763 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282771 2024-12-09T08:18:02,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,779 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282771) roll requested 2024-12-09T08:18:02,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282779 2024-12-09T08:18:02,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282771 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282779 2024-12-09T08:18:02,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282779) roll requested 2024-12-09T08:18:02,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282788 2024-12-09T08:18:02,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282788 2024-12-09T08:18:02,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282788) roll requested 2024-12-09T08:18:02,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282796 2024-12-09T08:18:02,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282796 2024-12-09T08:18:02,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,802 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282796) roll requested 2024-12-09T08:18:02,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282803 2024-12-09T08:18:02,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282803 2024-12-09T08:18:02,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,809 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282803) roll requested 2024-12-09T08:18:02,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282809 2024-12-09T08:18:02,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282809 2024-12-09T08:18:02,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282809) roll requested 2024-12-09T08:18:02,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282818 2024-12-09T08:18:02,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282809 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282818 2024-12-09T08:18:02,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,826 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282818) roll requested 2024-12-09T08:18:02,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282827 2024-12-09T08:18:02,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282827 2024-12-09T08:18:02,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282827) roll requested 2024-12-09T08:18:02,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282836 2024-12-09T08:18:02,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282836 2024-12-09T08:18:02,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,846 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282836) roll requested 2024-12-09T08:18:02,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282847 2024-12-09T08:18:02,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282847 2024-12-09T08:18:02,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,854 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282847) roll requested 2024-12-09T08:18:02,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282854 2024-12-09T08:18:02,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282854 2024-12-09T08:18:02,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282854) roll requested 2024-12-09T08:18:02,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282859 2024-12-09T08:18:02,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282859 2024-12-09T08:18:02,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282859) roll requested 2024-12-09T08:18:02,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282865 2024-12-09T08:18:02,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282859 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282865 2024-12-09T08:18:02,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,872 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282865) roll requested 2024-12-09T08:18:02,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282872 2024-12-09T08:18:02,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282872 2024-12-09T08:18:02,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282872) roll requested 2024-12-09T08:18:02,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282881 2024-12-09T08:18:02,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282872 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282881 2024-12-09T08:18:02,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282881) roll requested 2024-12-09T08:18:02,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282890 2024-12-09T08:18:02,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282890 2024-12-09T08:18:02,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282890) roll requested 2024-12-09T08:18:02,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282895 2024-12-09T08:18:02,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282890 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282895 2024-12-09T08:18:02,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282895) roll requested 2024-12-09T08:18:02,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282900 2024-12-09T08:18:02,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282900 2024-12-09T08:18:02,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282900) roll requested 2024-12-09T08:18:02,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282904 2024-12-09T08:18:02,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282904 2024-12-09T08:18:02,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,909 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282904) roll requested 2024-12-09T08:18:02,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282909 2024-12-09T08:18:02,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282904 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282909 2024-12-09T08:18:02,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,915 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282909) roll requested 2024-12-09T08:18:02,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282915 2024-12-09T08:18:02,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282915 2024-12-09T08:18:02,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,922 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282915) roll requested 2024-12-09T08:18:02,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282922 2024-12-09T08:18:02,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282922 2024-12-09T08:18:02,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,929 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282922) roll requested 2024-12-09T08:18:02,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282929 2024-12-09T08:18:02,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282922 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282929 2024-12-09T08:18:02,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282929) roll requested 2024-12-09T08:18:02,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282937 2024-12-09T08:18:02,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282937 2024-12-09T08:18:02,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282937) roll requested 2024-12-09T08:18:02,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282947 2024-12-09T08:18:02,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282947 2024-12-09T08:18:02,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,956 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282947) roll requested 2024-12-09T08:18:02,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282956 2024-12-09T08:18:02,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282956 2024-12-09T08:18:02,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,970 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282956) roll requested 2024-12-09T08:18:02,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282970 2024-12-09T08:18:02,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282956 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282970 2024-12-09T08:18:02,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,975 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282970) roll requested 2024-12-09T08:18:02,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282975 2024-12-09T08:18:02,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282970 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282975 2024-12-09T08:18:02,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282975) roll requested 2024-12-09T08:18:02,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282980 2024-12-09T08:18:02,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282975 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282980 2024-12-09T08:18:02,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,984 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282980) roll requested 2024-12-09T08:18:02,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282985 2024-12-09T08:18:02,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282985 2024-12-09T08:18:02,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,990 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282985) roll requested 2024-12-09T08:18:02,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282990 2024-12-09T08:18:02,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:02,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282990 2024-12-09T08:18:02,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:02,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:02,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282990) roll requested 2024-12-09T08:18:02,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732282996 2024-12-09T08:18:03,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282990 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282996 2024-12-09T08:18:03,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,002 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732282996) roll requested 2024-12-09T08:18:03,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283002 2024-12-09T08:18:03,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732282996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283002 2024-12-09T08:18:03,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283002) roll requested 2024-12-09T08:18:03,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283010 2024-12-09T08:18:03,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283002 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283010 2024-12-09T08:18:03,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283010) roll requested 2024-12-09T08:18:03,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283018 2024-12-09T08:18:03,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283018 2024-12-09T08:18:03,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283018) roll requested 2024-12-09T08:18:03,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283026 2024-12-09T08:18:03,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283026 2024-12-09T08:18:03,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,034 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283026) roll requested 2024-12-09T08:18:03,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283034 2024-12-09T08:18:03,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283034 2024-12-09T08:18:03,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283034) roll requested 2024-12-09T08:18:03,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283044 2024-12-09T08:18:03,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283044 2024-12-09T08:18:03,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,054 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283044) roll requested 2024-12-09T08:18:03,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283054 2024-12-09T08:18:03,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283054 2024-12-09T08:18:03,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283054) roll requested 2024-12-09T08:18:03,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283063 2024-12-09T08:18:03,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283063 2024-12-09T08:18:03,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283063) roll requested 2024-12-09T08:18:03,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283070 2024-12-09T08:18:03,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283063 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283070 2024-12-09T08:18:03,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283070) roll requested 2024-12-09T08:18:03,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283077 2024-12-09T08:18:03,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283077 2024-12-09T08:18:03,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,084 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283077) roll requested 2024-12-09T08:18:03,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283084 2024-12-09T08:18:03,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283084 2024-12-09T08:18:03,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,091 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283084) roll requested 2024-12-09T08:18:03,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283092 2024-12-09T08:18:03,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283084 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283092 2024-12-09T08:18:03,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283092) roll requested 2024-12-09T08:18:03,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283100 2024-12-09T08:18:03,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283092 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283100 2024-12-09T08:18:03,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283100) roll requested 2024-12-09T08:18:03,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283109 2024-12-09T08:18:03,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283109 2024-12-09T08:18:03,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,121 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283109) roll requested 2024-12-09T08:18:03,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283121 2024-12-09T08:18:03,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283121 2024-12-09T08:18:03,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283121) roll requested 2024-12-09T08:18:03,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283130 2024-12-09T08:18:03,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283130 2024-12-09T08:18:03,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283130) roll requested 2024-12-09T08:18:03,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283138 2024-12-09T08:18:03,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283130 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283138 2024-12-09T08:18:03,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283138) roll requested 2024-12-09T08:18:03,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283145 2024-12-09T08:18:03,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283138 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283145 2024-12-09T08:18:03,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283145) roll requested 2024-12-09T08:18:03,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283153 2024-12-09T08:18:03,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283153 2024-12-09T08:18:03,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,161 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283153) roll requested 2024-12-09T08:18:03,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283161 2024-12-09T08:18:03,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283161 2024-12-09T08:18:03,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,167 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283161) roll requested 2024-12-09T08:18:03,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283168 2024-12-09T08:18:03,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283168 2024-12-09T08:18:03,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,174 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283168) roll requested 2024-12-09T08:18:03,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283175 2024-12-09T08:18:03,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283175 2024-12-09T08:18:03,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,181 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283175) roll requested 2024-12-09T08:18:03,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283181 2024-12-09T08:18:03,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283175 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283181 2024-12-09T08:18:03,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283181) roll requested 2024-12-09T08:18:03,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283189 2024-12-09T08:18:03,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283189 2024-12-09T08:18:03,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283189) roll requested 2024-12-09T08:18:03,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283196 2024-12-09T08:18:03,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283196 2024-12-09T08:18:03,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283196) roll requested 2024-12-09T08:18:03,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283201 2024-12-09T08:18:03,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283201 2024-12-09T08:18:03,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,207 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283201) roll requested 2024-12-09T08:18:03,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283208 2024-12-09T08:18:03,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283208 2024-12-09T08:18:03,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,215 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283208) roll requested 2024-12-09T08:18:03,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283215 2024-12-09T08:18:03,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283215 2024-12-09T08:18:03,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,223 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283215) roll requested 2024-12-09T08:18:03,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283223 2024-12-09T08:18:03,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283223 2024-12-09T08:18:03,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283223) roll requested 2024-12-09T08:18:03,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283233 2024-12-09T08:18:03,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283223 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283233 2024-12-09T08:18:03,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,241 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283233) roll requested 2024-12-09T08:18:03,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283242 2024-12-09T08:18:03,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283242 2024-12-09T08:18:03,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283242) roll requested 2024-12-09T08:18:03,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283249 2024-12-09T08:18:03,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283249 2024-12-09T08:18:03,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283249) roll requested 2024-12-09T08:18:03,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283257 2024-12-09T08:18:03,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283257 2024-12-09T08:18:03,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283257) roll requested 2024-12-09T08:18:03,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283263 2024-12-09T08:18:03,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283263 2024-12-09T08:18:03,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283263) roll requested 2024-12-09T08:18:03,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283268 2024-12-09T08:18:03,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283268 2024-12-09T08:18:03,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,274 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283268) roll requested 2024-12-09T08:18:03,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283275 2024-12-09T08:18:03,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283275 2024-12-09T08:18:03,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,282 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283275) roll requested 2024-12-09T08:18:03,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283282 2024-12-09T08:18:03,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283282 2024-12-09T08:18:03,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283282) roll requested 2024-12-09T08:18:03,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283289 2024-12-09T08:18:03,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283282 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283289 2024-12-09T08:18:03,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,295 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283289) roll requested 2024-12-09T08:18:03,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283295 2024-12-09T08:18:03,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283295 2024-12-09T08:18:03,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283295) roll requested 2024-12-09T08:18:03,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283304 2024-12-09T08:18:03,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283295 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283304 2024-12-09T08:18:03,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283304) roll requested 2024-12-09T08:18:03,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283312 2024-12-09T08:18:03,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283312 2024-12-09T08:18:03,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283312) roll requested 2024-12-09T08:18:03,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283320 2024-12-09T08:18:03,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283320 2024-12-09T08:18:03,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283320) roll requested 2024-12-09T08:18:03,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283328 2024-12-09T08:18:03,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283328 2024-12-09T08:18:03,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283328) roll requested 2024-12-09T08:18:03,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283336 2024-12-09T08:18:03,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283336 2024-12-09T08:18:03,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283336) roll requested 2024-12-09T08:18:03,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283345 2024-12-09T08:18:03,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283345 2024-12-09T08:18:03,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,351 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283345) roll requested 2024-12-09T08:18:03,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283351 2024-12-09T08:18:03,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283351 2024-12-09T08:18:03,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,359 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283351) roll requested 2024-12-09T08:18:03,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283359 2024-12-09T08:18:03,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283351 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283359 2024-12-09T08:18:03,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,367 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,367 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283359) roll requested 2024-12-09T08:18:03,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283367 2024-12-09T08:18:03,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283367 2024-12-09T08:18:03,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283367) roll requested 2024-12-09T08:18:03,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283378 2024-12-09T08:18:03,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283367 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283378 2024-12-09T08:18:03,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,386 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283378) roll requested 2024-12-09T08:18:03,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283387 2024-12-09T08:18:03,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283387 2024-12-09T08:18:03,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,395 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283387) roll requested 2024-12-09T08:18:03,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283395 2024-12-09T08:18:03,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283395 2024-12-09T08:18:03,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,403 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283395) roll requested 2024-12-09T08:18:03,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283404 2024-12-09T08:18:03,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283395 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283404 2024-12-09T08:18:03,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283404) roll requested 2024-12-09T08:18:03,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283413 2024-12-09T08:18:03,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283413 2024-12-09T08:18:03,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283413) roll requested 2024-12-09T08:18:03,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283422 2024-12-09T08:18:03,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283422 2024-12-09T08:18:03,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,429 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283422) roll requested 2024-12-09T08:18:03,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283430 2024-12-09T08:18:03,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283430 2024-12-09T08:18:03,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283430) roll requested 2024-12-09T08:18:03,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283438 2024-12-09T08:18:03,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283438 2024-12-09T08:18:03,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283438) roll requested 2024-12-09T08:18:03,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283447 2024-12-09T08:18:03,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283438 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283447 2024-12-09T08:18:03,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283447) roll requested 2024-12-09T08:18:03,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283456 2024-12-09T08:18:03,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283456 2024-12-09T08:18:03,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283456) roll requested 2024-12-09T08:18:03,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283466 2024-12-09T08:18:03,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283466 2024-12-09T08:18:03,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283466) roll requested 2024-12-09T08:18:03,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283476 2024-12-09T08:18:03,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283476 2024-12-09T08:18:03,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283476) roll requested 2024-12-09T08:18:03,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283481 2024-12-09T08:18:03,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283481 2024-12-09T08:18:03,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283481) roll requested 2024-12-09T08:18:03,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283486 2024-12-09T08:18:03,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283486 2024-12-09T08:18:03,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,491 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283486) roll requested 2024-12-09T08:18:03,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283491 2024-12-09T08:18:03,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283486 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283491 2024-12-09T08:18:03,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,497 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283491) roll requested 2024-12-09T08:18:03,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283497 2024-12-09T08:18:03,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283497 2024-12-09T08:18:03,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,503 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283497) roll requested 2024-12-09T08:18:03,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283503 2024-12-09T08:18:03,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283503 2024-12-09T08:18:03,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,510 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283503) roll requested 2024-12-09T08:18:03,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283510 2024-12-09T08:18:03,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283510 2024-12-09T08:18:03,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,517 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283510) roll requested 2024-12-09T08:18:03,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283518 2024-12-09T08:18:03,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283510 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283518 2024-12-09T08:18:03,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283518) roll requested 2024-12-09T08:18:03,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283526 2024-12-09T08:18:03,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283518 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283526 2024-12-09T08:18:03,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,534 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283526) roll requested 2024-12-09T08:18:03,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283534 2024-12-09T08:18:03,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283534 2024-12-09T08:18:03,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283534) roll requested 2024-12-09T08:18:03,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283543 2024-12-09T08:18:03,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283534 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283543 2024-12-09T08:18:03,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283543) roll requested 2024-12-09T08:18:03,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283553 2024-12-09T08:18:03,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283553 2024-12-09T08:18:03,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283553) roll requested 2024-12-09T08:18:03,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283564 2024-12-09T08:18:03,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283564 2024-12-09T08:18:03,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,574 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283564) roll requested 2024-12-09T08:18:03,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283575 2024-12-09T08:18:03,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283575 2024-12-09T08:18:03,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,582 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283575) roll requested 2024-12-09T08:18:03,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283582 2024-12-09T08:18:03,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283575 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283582 2024-12-09T08:18:03,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283582) roll requested 2024-12-09T08:18:03,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283591 2024-12-09T08:18:03,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283582 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283591 2024-12-09T08:18:03,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283591) roll requested 2024-12-09T08:18:03,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283599 2024-12-09T08:18:03,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283591 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283599 2024-12-09T08:18:03,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283599) roll requested 2024-12-09T08:18:03,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283608 2024-12-09T08:18:03,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283608 2024-12-09T08:18:03,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,618 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283608) roll requested 2024-12-09T08:18:03,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283619 2024-12-09T08:18:03,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283619 2024-12-09T08:18:03,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283619) roll requested 2024-12-09T08:18:03,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283628 2024-12-09T08:18:03,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283628 2024-12-09T08:18:03,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283628) roll requested 2024-12-09T08:18:03,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283637 2024-12-09T08:18:03,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283637 2024-12-09T08:18:03,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,643 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283637) roll requested 2024-12-09T08:18:03,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283644 2024-12-09T08:18:03,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283637 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283644 2024-12-09T08:18:03,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283644) roll requested 2024-12-09T08:18:03,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283651 2024-12-09T08:18:03,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283651 2024-12-09T08:18:03,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,657 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283651) roll requested 2024-12-09T08:18:03,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283657 2024-12-09T08:18:03,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283657 2024-12-09T08:18:03,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283657) roll requested 2024-12-09T08:18:03,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283664 2024-12-09T08:18:03,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283664 2024-12-09T08:18:03,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,671 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283664) roll requested 2024-12-09T08:18:03,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283672 2024-12-09T08:18:03,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283672 2024-12-09T08:18:03,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283672) roll requested 2024-12-09T08:18:03,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283680 2024-12-09T08:18:03,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283680 2024-12-09T08:18:03,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,687 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283680) roll requested 2024-12-09T08:18:03,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283688 2024-12-09T08:18:03,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283680 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283688 2024-12-09T08:18:03,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283688) roll requested 2024-12-09T08:18:03,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283694 2024-12-09T08:18:03,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283688 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283694 2024-12-09T08:18:03,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,702 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283694) roll requested 2024-12-09T08:18:03,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283702 2024-12-09T08:18:03,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283694 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283702 2024-12-09T08:18:03,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,710 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283702) roll requested 2024-12-09T08:18:03,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283710 2024-12-09T08:18:03,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283710 2024-12-09T08:18:03,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283710) roll requested 2024-12-09T08:18:03,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283719 2024-12-09T08:18:03,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283710 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283719 2024-12-09T08:18:03,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283719) roll requested 2024-12-09T08:18:03,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283727 2024-12-09T08:18:03,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283727 2024-12-09T08:18:03,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283727) roll requested 2024-12-09T08:18:03,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283736 2024-12-09T08:18:03,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283736 2024-12-09T08:18:03,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283736) roll requested 2024-12-09T08:18:03,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283745 2024-12-09T08:18:03,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283745 2024-12-09T08:18:03,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283745) roll requested 2024-12-09T08:18:03,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283750 2024-12-09T08:18:03,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283750 2024-12-09T08:18:03,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,755 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283750) roll requested 2024-12-09T08:18:03,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283755 2024-12-09T08:18:03,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283755 2024-12-09T08:18:03,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283755) roll requested 2024-12-09T08:18:03,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283761 2024-12-09T08:18:03,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283755 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283761 2024-12-09T08:18:03,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283761) roll requested 2024-12-09T08:18:03,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283769 2024-12-09T08:18:03,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283769 2024-12-09T08:18:03,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,774 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283769) roll requested 2024-12-09T08:18:03,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283775 2024-12-09T08:18:03,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283775 2024-12-09T08:18:03,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283775) roll requested 2024-12-09T08:18:03,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283780 2024-12-09T08:18:03,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283780 2024-12-09T08:18:03,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283780) roll requested 2024-12-09T08:18:03,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283791 2024-12-09T08:18:03,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283791 2024-12-09T08:18:03,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,797 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283791) roll requested 2024-12-09T08:18:03,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283797 2024-12-09T08:18:03,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283797 2024-12-09T08:18:03,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283797) roll requested 2024-12-09T08:18:03,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283804 2024-12-09T08:18:03,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283804 2024-12-09T08:18:03,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,811 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283804) roll requested 2024-12-09T08:18:03,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283811 2024-12-09T08:18:03,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283811 2024-12-09T08:18:03,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283811) roll requested 2024-12-09T08:18:03,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283816 2024-12-09T08:18:03,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283811 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283816 2024-12-09T08:18:03,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283816) roll requested 2024-12-09T08:18:03,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283821 2024-12-09T08:18:03,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283816 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283821 2024-12-09T08:18:03,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283821) roll requested 2024-12-09T08:18:03,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283828 2024-12-09T08:18:03,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283828 2024-12-09T08:18:03,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,834 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283828) roll requested 2024-12-09T08:18:03,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283834 2024-12-09T08:18:03,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283828 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283834 2024-12-09T08:18:03,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283834) roll requested 2024-12-09T08:18:03,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283841 2024-12-09T08:18:03,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283841 2024-12-09T08:18:03,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283841) roll requested 2024-12-09T08:18:03,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283849 2024-12-09T08:18:03,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283849 2024-12-09T08:18:03,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,858 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283849) roll requested 2024-12-09T08:18:03,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283858 2024-12-09T08:18:03,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283858 2024-12-09T08:18:03,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,863 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283858) roll requested 2024-12-09T08:18:03,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283863 2024-12-09T08:18:03,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283858 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283863 2024-12-09T08:18:03,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,869 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283863) roll requested 2024-12-09T08:18:03,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283870 2024-12-09T08:18:03,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283863 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283870 2024-12-09T08:18:03,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283870) roll requested 2024-12-09T08:18:03,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283877 2024-12-09T08:18:03,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283870 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283877 2024-12-09T08:18:03,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283877) roll requested 2024-12-09T08:18:03,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283885 2024-12-09T08:18:03,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283877 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283885 2024-12-09T08:18:03,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,892 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283885) roll requested 2024-12-09T08:18:03,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283892 2024-12-09T08:18:03,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283892 2024-12-09T08:18:03,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283892) roll requested 2024-12-09T08:18:03,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283899 2024-12-09T08:18:03,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283892 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283899 2024-12-09T08:18:03,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283899) roll requested 2024-12-09T08:18:03,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283906 2024-12-09T08:18:03,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283899 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283906 2024-12-09T08:18:03,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,915 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283906) roll requested 2024-12-09T08:18:03,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283915 2024-12-09T08:18:03,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283906 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283915 2024-12-09T08:18:03,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,923 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283915) roll requested 2024-12-09T08:18:03,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283923 2024-12-09T08:18:03,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283923 2024-12-09T08:18:03,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,930 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283923) roll requested 2024-12-09T08:18:03,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283930 2024-12-09T08:18:03,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283923 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283930 2024-12-09T08:18:03,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,938 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283930) roll requested 2024-12-09T08:18:03,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283938 2024-12-09T08:18:03,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283930 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283938 2024-12-09T08:18:03,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283938) roll requested 2024-12-09T08:18:03,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283946 2024-12-09T08:18:03,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283938 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283946 2024-12-09T08:18:03,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,953 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283946) roll requested 2024-12-09T08:18:03,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283953 2024-12-09T08:18:03,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283946 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283953 2024-12-09T08:18:03,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283953) roll requested 2024-12-09T08:18:03,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283960 2024-12-09T08:18:03,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283960 2024-12-09T08:18:03,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,967 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283960) roll requested 2024-12-09T08:18:03,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283967 2024-12-09T08:18:03,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283967 2024-12-09T08:18:03,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,974 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283967) roll requested 2024-12-09T08:18:03,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283974 2024-12-09T08:18:03,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283967 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283974 2024-12-09T08:18:03,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283974) roll requested 2024-12-09T08:18:03,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283981 2024-12-09T08:18:03,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283974 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283981 2024-12-09T08:18:03,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283981) roll requested 2024-12-09T08:18:03,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283989 2024-12-09T08:18:03,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:03,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283989 2024-12-09T08:18:03,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:03,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:03,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283989) roll requested 2024-12-09T08:18:03,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732283997 2024-12-09T08:18:04,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283997 2024-12-09T08:18:04,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732283997) roll requested 2024-12-09T08:18:04,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284004 2024-12-09T08:18:04,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732283997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284004 2024-12-09T08:18:04,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284004) roll requested 2024-12-09T08:18:04,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284012 2024-12-09T08:18:04,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284012 2024-12-09T08:18:04,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,029 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284012) roll requested 2024-12-09T08:18:04,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284029 2024-12-09T08:18:04,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284029 2024-12-09T08:18:04,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284029) roll requested 2024-12-09T08:18:04,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284037 2024-12-09T08:18:04,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284037 2024-12-09T08:18:04,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,042 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284037) roll requested 2024-12-09T08:18:04,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284042 2024-12-09T08:18:04,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284042 2024-12-09T08:18:04,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284042) roll requested 2024-12-09T08:18:04,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284047 2024-12-09T08:18:04,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284042 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284047 2024-12-09T08:18:04,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284047) roll requested 2024-12-09T08:18:04,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284052 2024-12-09T08:18:04,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284052 2024-12-09T08:18:04,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,058 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284052) roll requested 2024-12-09T08:18:04,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284058 2024-12-09T08:18:04,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284058 2024-12-09T08:18:04,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,065 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284058) roll requested 2024-12-09T08:18:04,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284066 2024-12-09T08:18:04,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284058 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284066 2024-12-09T08:18:04,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284066) roll requested 2024-12-09T08:18:04,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284073 2024-12-09T08:18:04,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284066 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284073 2024-12-09T08:18:04,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284073) roll requested 2024-12-09T08:18:04,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284081 2024-12-09T08:18:04,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284081 2024-12-09T08:18:04,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284081) roll requested 2024-12-09T08:18:04,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284089 2024-12-09T08:18:04,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284081 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284089 2024-12-09T08:18:04,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,097 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284089) roll requested 2024-12-09T08:18:04,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284097 2024-12-09T08:18:04,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284097 2024-12-09T08:18:04,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284097) roll requested 2024-12-09T08:18:04,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284106 2024-12-09T08:18:04,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284106 2024-12-09T08:18:04,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,115 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284106) roll requested 2024-12-09T08:18:04,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284115 2024-12-09T08:18:04,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284115 2024-12-09T08:18:04,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284115) roll requested 2024-12-09T08:18:04,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284123 2024-12-09T08:18:04,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284115 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284123 2024-12-09T08:18:04,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,131 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284123) roll requested 2024-12-09T08:18:04,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284131 2024-12-09T08:18:04,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284123 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284131 2024-12-09T08:18:04,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284131) roll requested 2024-12-09T08:18:04,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284141 2024-12-09T08:18:04,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284141 2024-12-09T08:18:04,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284141) roll requested 2024-12-09T08:18:04,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284151 2024-12-09T08:18:04,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284151 2024-12-09T08:18:04,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284151) roll requested 2024-12-09T08:18:04,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284160 2024-12-09T08:18:04,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284160 2024-12-09T08:18:04,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,167 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284160) roll requested 2024-12-09T08:18:04,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284167 2024-12-09T08:18:04,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284167 2024-12-09T08:18:04,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284167) roll requested 2024-12-09T08:18:04,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284172 2024-12-09T08:18:04,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284167 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284172 2024-12-09T08:18:04,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284172) roll requested 2024-12-09T08:18:04,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284176 2024-12-09T08:18:04,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284176 2024-12-09T08:18:04,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,182 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284176) roll requested 2024-12-09T08:18:04,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284183 2024-12-09T08:18:04,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284183 2024-12-09T08:18:04,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284183) roll requested 2024-12-09T08:18:04,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284189 2024-12-09T08:18:04,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284189 2024-12-09T08:18:04,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284189) roll requested 2024-12-09T08:18:04,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284196 2024-12-09T08:18:04,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284196 2024-12-09T08:18:04,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284196) roll requested 2024-12-09T08:18:04,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284205 2024-12-09T08:18:04,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284205 2024-12-09T08:18:04,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284205) roll requested 2024-12-09T08:18:04,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284212 2024-12-09T08:18:04,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284212 2024-12-09T08:18:04,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284212) roll requested 2024-12-09T08:18:04,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284218 2024-12-09T08:18:04,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284218 2024-12-09T08:18:04,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284218) roll requested 2024-12-09T08:18:04,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284225 2024-12-09T08:18:04,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284218 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284225 2024-12-09T08:18:04,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284225) roll requested 2024-12-09T08:18:04,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284231 2024-12-09T08:18:04,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284231 2024-12-09T08:18:04,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284231) roll requested 2024-12-09T08:18:04,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284239 2024-12-09T08:18:04,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284239 2024-12-09T08:18:04,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284239) roll requested 2024-12-09T08:18:04,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284246 2024-12-09T08:18:04,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284239 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284246 2024-12-09T08:18:04,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,253 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284246) roll requested 2024-12-09T08:18:04,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284253 2024-12-09T08:18:04,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284253 2024-12-09T08:18:04,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,260 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284253) roll requested 2024-12-09T08:18:04,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284260 2024-12-09T08:18:04,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284260 2024-12-09T08:18:04,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284260) roll requested 2024-12-09T08:18:04,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284267 2024-12-09T08:18:04,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284260 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284267 2024-12-09T08:18:04,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,274 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284267) roll requested 2024-12-09T08:18:04,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284274 2024-12-09T08:18:04,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284274 2024-12-09T08:18:04,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284274) roll requested 2024-12-09T08:18:04,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284281 2024-12-09T08:18:04,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284274 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284281 2024-12-09T08:18:04,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284281) roll requested 2024-12-09T08:18:04,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284289 2024-12-09T08:18:04,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284289 2024-12-09T08:18:04,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,298 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284289) roll requested 2024-12-09T08:18:04,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284298 2024-12-09T08:18:04,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284298 2024-12-09T08:18:04,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284298) roll requested 2024-12-09T08:18:04,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284306 2024-12-09T08:18:04,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284306 2024-12-09T08:18:04,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284306) roll requested 2024-12-09T08:18:04,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284312 2024-12-09T08:18:04,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284312 2024-12-09T08:18:04,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284312) roll requested 2024-12-09T08:18:04,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284320 2024-12-09T08:18:04,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284320 2024-12-09T08:18:04,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284320) roll requested 2024-12-09T08:18:04,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284327 2024-12-09T08:18:04,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284327 2024-12-09T08:18:04,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,335 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284327) roll requested 2024-12-09T08:18:04,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284335 2024-12-09T08:18:04,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284335 2024-12-09T08:18:04,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284335) roll requested 2024-12-09T08:18:04,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284345 2024-12-09T08:18:04,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284345 2024-12-09T08:18:04,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284345) roll requested 2024-12-09T08:18:04,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284353 2024-12-09T08:18:04,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284353 2024-12-09T08:18:04,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284353) roll requested 2024-12-09T08:18:04,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284360 2024-12-09T08:18:04,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284360 2024-12-09T08:18:04,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,365 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284360) roll requested 2024-12-09T08:18:04,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284365 2024-12-09T08:18:04,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284360 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284365 2024-12-09T08:18:04,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284365) roll requested 2024-12-09T08:18:04,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284370 2024-12-09T08:18:04,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284365 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284370 2024-12-09T08:18:04,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284370) roll requested 2024-12-09T08:18:04,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284375 2024-12-09T08:18:04,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284375 2024-12-09T08:18:04,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,380 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284375) roll requested 2024-12-09T08:18:04,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284380 2024-12-09T08:18:04,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284375 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284380 2024-12-09T08:18:04,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,385 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284380) roll requested 2024-12-09T08:18:04,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284385 2024-12-09T08:18:04,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284385 2024-12-09T08:18:04,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,393 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284385) roll requested 2024-12-09T08:18:04,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284393 2024-12-09T08:18:04,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284393 2024-12-09T08:18:04,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,398 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284393) roll requested 2024-12-09T08:18:04,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284398 2024-12-09T08:18:04,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284398 2024-12-09T08:18:04,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,403 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284398) roll requested 2024-12-09T08:18:04,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284403 2024-12-09T08:18:04,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284398 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284403 2024-12-09T08:18:04,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,408 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284403) roll requested 2024-12-09T08:18:04,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284408 2024-12-09T08:18:04,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284403 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284408 2024-12-09T08:18:04,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284408) roll requested 2024-12-09T08:18:04,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284413 2024-12-09T08:18:04,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284408 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284413 2024-12-09T08:18:04,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,418 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284413) roll requested 2024-12-09T08:18:04,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284418 2024-12-09T08:18:04,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284418 2024-12-09T08:18:04,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284418) roll requested 2024-12-09T08:18:04,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284425 2024-12-09T08:18:04,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284418 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284425 2024-12-09T08:18:04,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,430 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284425) roll requested 2024-12-09T08:18:04,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284430 2024-12-09T08:18:04,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284430 2024-12-09T08:18:04,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,435 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284430) roll requested 2024-12-09T08:18:04,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284435 2024-12-09T08:18:04,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284435 2024-12-09T08:18:04,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284435) roll requested 2024-12-09T08:18:04,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284440 2024-12-09T08:18:04,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284435 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284440 2024-12-09T08:18:04,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284440) roll requested 2024-12-09T08:18:04,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284445 2024-12-09T08:18:04,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284445 2024-12-09T08:18:04,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,455 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284445) roll requested 2024-12-09T08:18:04,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284455 2024-12-09T08:18:04,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284455 2024-12-09T08:18:04,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284455) roll requested 2024-12-09T08:18:04,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284461 2024-12-09T08:18:04,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284455 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284461 2024-12-09T08:18:04,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284461) roll requested 2024-12-09T08:18:04,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284466 2024-12-09T08:18:04,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284466 2024-12-09T08:18:04,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284466) roll requested 2024-12-09T08:18:04,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284471 2024-12-09T08:18:04,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284471 2024-12-09T08:18:04,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284471) roll requested 2024-12-09T08:18:04,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284476 2024-12-09T08:18:04,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284476 2024-12-09T08:18:04,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284476) roll requested 2024-12-09T08:18:04,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284481 2024-12-09T08:18:04,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284481 2024-12-09T08:18:04,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284481) roll requested 2024-12-09T08:18:04,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284487 2024-12-09T08:18:04,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284487 2024-12-09T08:18:04,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,491 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284487) roll requested 2024-12-09T08:18:04,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284491 2024-12-09T08:18:04,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284491 2024-12-09T08:18:04,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284491) roll requested 2024-12-09T08:18:04,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284496 2024-12-09T08:18:04,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284496 2024-12-09T08:18:04,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284496) roll requested 2024-12-09T08:18:04,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284502 2024-12-09T08:18:04,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284502 2024-12-09T08:18:04,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284502) roll requested 2024-12-09T08:18:04,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284507 2024-12-09T08:18:04,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284507 2024-12-09T08:18:04,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284507) roll requested 2024-12-09T08:18:04,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284512 2024-12-09T08:18:04,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284512 2024-12-09T08:18:04,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,517 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284512) roll requested 2024-12-09T08:18:04,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284517 2024-12-09T08:18:04,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284517 2024-12-09T08:18:04,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284517) roll requested 2024-12-09T08:18:04,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284523 2024-12-09T08:18:04,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284523 2024-12-09T08:18:04,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284523) roll requested 2024-12-09T08:18:04,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284528 2024-12-09T08:18:04,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284523 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284528 2024-12-09T08:18:04,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,532 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284528) roll requested 2024-12-09T08:18:04,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284533 2024-12-09T08:18:04,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284533 2024-12-09T08:18:04,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,538 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284533) roll requested 2024-12-09T08:18:04,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284538 2024-12-09T08:18:04,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284538 2024-12-09T08:18:04,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284538) roll requested 2024-12-09T08:18:04,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284543 2024-12-09T08:18:04,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284538 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284543 2024-12-09T08:18:04,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284543) roll requested 2024-12-09T08:18:04,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284549 2024-12-09T08:18:04,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284549 2024-12-09T08:18:04,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,554 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284549) roll requested 2024-12-09T08:18:04,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284554 2024-12-09T08:18:04,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284554 2024-12-09T08:18:04,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284554) roll requested 2024-12-09T08:18:04,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284559 2024-12-09T08:18:04,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284554 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284559 2024-12-09T08:18:04,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284559) roll requested 2024-12-09T08:18:04,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284565 2024-12-09T08:18:04,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284565 2024-12-09T08:18:04,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284565) roll requested 2024-12-09T08:18:04,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284569 2024-12-09T08:18:04,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284569 2024-12-09T08:18:04,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,574 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284569) roll requested 2024-12-09T08:18:04,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284575 2024-12-09T08:18:04,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284575 2024-12-09T08:18:04,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284575) roll requested 2024-12-09T08:18:04,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284580 2024-12-09T08:18:04,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284575 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284580 2024-12-09T08:18:04,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,586 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284580) roll requested 2024-12-09T08:18:04,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284586 2024-12-09T08:18:04,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284586 2024-12-09T08:18:04,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,594 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284586) roll requested 2024-12-09T08:18:04,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284594 2024-12-09T08:18:04,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284586 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284594 2024-12-09T08:18:04,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,602 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284594) roll requested 2024-12-09T08:18:04,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284602 2024-12-09T08:18:04,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284602 2024-12-09T08:18:04,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,610 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284602) roll requested 2024-12-09T08:18:04,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284610 2024-12-09T08:18:04,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284610 2024-12-09T08:18:04,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284610) roll requested 2024-12-09T08:18:04,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284620 2024-12-09T08:18:04,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284610 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284620 2024-12-09T08:18:04,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284620) roll requested 2024-12-09T08:18:04,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284628 2024-12-09T08:18:04,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284628 2024-12-09T08:18:04,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,635 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284628) roll requested 2024-12-09T08:18:04,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284635 2024-12-09T08:18:04,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284635 2024-12-09T08:18:04,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284635) roll requested 2024-12-09T08:18:04,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284642 2024-12-09T08:18:04,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284635 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284642 2024-12-09T08:18:04,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284642) roll requested 2024-12-09T08:18:04,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284650 2024-12-09T08:18:04,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284642 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284650 2024-12-09T08:18:04,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284650) roll requested 2024-12-09T08:18:04,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284656 2024-12-09T08:18:04,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284650 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284656 2024-12-09T08:18:04,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284656) roll requested 2024-12-09T08:18:04,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284664 2024-12-09T08:18:04,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284664 2024-12-09T08:18:04,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,671 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284664) roll requested 2024-12-09T08:18:04,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284671 2024-12-09T08:18:04,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284671 2024-12-09T08:18:04,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284671) roll requested 2024-12-09T08:18:04,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284677 2024-12-09T08:18:04,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284671 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284677 2024-12-09T08:18:04,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284677) roll requested 2024-12-09T08:18:04,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284683 2024-12-09T08:18:04,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284683 2024-12-09T08:18:04,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284683) roll requested 2024-12-09T08:18:04,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284690 2024-12-09T08:18:04,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284690 2024-12-09T08:18:04,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284690) roll requested 2024-12-09T08:18:04,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284698 2024-12-09T08:18:04,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284698 2024-12-09T08:18:04,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,706 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284698) roll requested 2024-12-09T08:18:04,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284706 2024-12-09T08:18:04,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284698 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284706 2024-12-09T08:18:04,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,715 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284706) roll requested 2024-12-09T08:18:04,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284715 2024-12-09T08:18:04,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284706 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284715 2024-12-09T08:18:04,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,723 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284715) roll requested 2024-12-09T08:18:04,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284723 2024-12-09T08:18:04,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284723 2024-12-09T08:18:04,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284723) roll requested 2024-12-09T08:18:04,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284732 2024-12-09T08:18:04,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284723 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284732 2024-12-09T08:18:04,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,741 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284732) roll requested 2024-12-09T08:18:04,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284741 2024-12-09T08:18:04,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284741 2024-12-09T08:18:04,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284741) roll requested 2024-12-09T08:18:04,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284750 2024-12-09T08:18:04,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284741 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284750 2024-12-09T08:18:04,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,758 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284750) roll requested 2024-12-09T08:18:04,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284759 2024-12-09T08:18:04,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284759 2024-12-09T08:18:04,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,766 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284759) roll requested 2024-12-09T08:18:04,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284766 2024-12-09T08:18:04,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284766 2024-12-09T08:18:04,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,773 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284766) roll requested 2024-12-09T08:18:04,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284773 2024-12-09T08:18:04,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284766 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284773 2024-12-09T08:18:04,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,781 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284773) roll requested 2024-12-09T08:18:04,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284781 2024-12-09T08:18:04,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284781 2024-12-09T08:18:04,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,790 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284781) roll requested 2024-12-09T08:18:04,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284790 2024-12-09T08:18:04,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284781 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284790 2024-12-09T08:18:04,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,798 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284790) roll requested 2024-12-09T08:18:04,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284799 2024-12-09T08:18:04,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284790 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284799 2024-12-09T08:18:04,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,807 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284799) roll requested 2024-12-09T08:18:04,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284807 2024-12-09T08:18:04,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284807 2024-12-09T08:18:04,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284807) roll requested 2024-12-09T08:18:04,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284816 2024-12-09T08:18:04,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284807 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284816 2024-12-09T08:18:04,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284816) roll requested 2024-12-09T08:18:04,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284821 2024-12-09T08:18:04,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284816 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284821 2024-12-09T08:18:04,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,826 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284821) roll requested 2024-12-09T08:18:04,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284826 2024-12-09T08:18:04,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284826 2024-12-09T08:18:04,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284826) roll requested 2024-12-09T08:18:04,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284832 2024-12-09T08:18:04,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284832 2024-12-09T08:18:04,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,838 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284832) roll requested 2024-12-09T08:18:04,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284839 2024-12-09T08:18:04,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284839 2024-12-09T08:18:04,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284839) roll requested 2024-12-09T08:18:04,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284848 2024-12-09T08:18:04,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284839 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284848 2024-12-09T08:18:04,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284848) roll requested 2024-12-09T08:18:04,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284857 2024-12-09T08:18:04,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284857 2024-12-09T08:18:04,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284857) roll requested 2024-12-09T08:18:04,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284862 2024-12-09T08:18:04,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284862 2024-12-09T08:18:04,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284862) roll requested 2024-12-09T08:18:04,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284868 2024-12-09T08:18:04,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284868 2024-12-09T08:18:04,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284868) roll requested 2024-12-09T08:18:04,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284877 2024-12-09T08:18:04,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284877 2024-12-09T08:18:04,882 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,882 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284877) roll requested 2024-12-09T08:18:04,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284882 2024-12-09T08:18:04,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284877 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284882 2024-12-09T08:18:04,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,889 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284882) roll requested 2024-12-09T08:18:04,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284889 2024-12-09T08:18:04,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284882 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284889 2024-12-09T08:18:04,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284889) roll requested 2024-12-09T08:18:04,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284893 2024-12-09T08:18:04,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284893 2024-12-09T08:18:04,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284893) roll requested 2024-12-09T08:18:04,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284901 2024-12-09T08:18:04,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284901 2024-12-09T08:18:04,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284901) roll requested 2024-12-09T08:18:04,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284906 2024-12-09T08:18:04,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284906 2024-12-09T08:18:04,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,911 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284906) roll requested 2024-12-09T08:18:04,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284911 2024-12-09T08:18:04,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284906 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284911 2024-12-09T08:18:04,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284911) roll requested 2024-12-09T08:18:04,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284916 2024-12-09T08:18:04,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284916 2024-12-09T08:18:04,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284916) roll requested 2024-12-09T08:18:04,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284921 2024-12-09T08:18:04,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284921 2024-12-09T08:18:04,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284921) roll requested 2024-12-09T08:18:04,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284929 2024-12-09T08:18:04,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284929 2024-12-09T08:18:04,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284929) roll requested 2024-12-09T08:18:04,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284937 2024-12-09T08:18:04,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284937 2024-12-09T08:18:04,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,942 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284937) roll requested 2024-12-09T08:18:04,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284942 2024-12-09T08:18:04,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284942 2024-12-09T08:18:04,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284942) roll requested 2024-12-09T08:18:04,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284947 2024-12-09T08:18:04,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284947 2024-12-09T08:18:04,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284947) roll requested 2024-12-09T08:18:04,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284952 2024-12-09T08:18:04,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284952 2024-12-09T08:18:04,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284952) roll requested 2024-12-09T08:18:04,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284958 2024-12-09T08:18:04,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284958 2024-12-09T08:18:04,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,963 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284958) roll requested 2024-12-09T08:18:04,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284963 2024-12-09T08:18:04,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284963 2024-12-09T08:18:04,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284963) roll requested 2024-12-09T08:18:04,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284969 2024-12-09T08:18:04,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284963 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284969 2024-12-09T08:18:04,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284969) roll requested 2024-12-09T08:18:04,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284976 2024-12-09T08:18:04,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284976 2024-12-09T08:18:04,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284976) roll requested 2024-12-09T08:18:04,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284983 2024-12-09T08:18:04,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:04,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284983 2024-12-09T08:18:04,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:04,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:04,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284983) roll requested 2024-12-09T08:18:04,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732284993 2024-12-09T08:18:05,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284993 2024-12-09T08:18:05,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732284993) roll requested 2024-12-09T08:18:05,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285002 2024-12-09T08:18:05,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732284993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285002 2024-12-09T08:18:05,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285002) roll requested 2024-12-09T08:18:05,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285009 2024-12-09T08:18:05,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285002 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285009 2024-12-09T08:18:05,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285009) roll requested 2024-12-09T08:18:05,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285021 2024-12-09T08:18:05,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285021 2024-12-09T08:18:05,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,027 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285021) roll requested 2024-12-09T08:18:05,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285027 2024-12-09T08:18:05,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285027 2024-12-09T08:18:05,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285027) roll requested 2024-12-09T08:18:05,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285033 2024-12-09T08:18:05,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285027 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285033 2024-12-09T08:18:05,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285033) roll requested 2024-12-09T08:18:05,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285040 2024-12-09T08:18:05,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285040 2024-12-09T08:18:05,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,046 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285040) roll requested 2024-12-09T08:18:05,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285046 2024-12-09T08:18:05,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285046 2024-12-09T08:18:05,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285046) roll requested 2024-12-09T08:18:05,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285052 2024-12-09T08:18:05,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285046 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285052 2024-12-09T08:18:05,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,060 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285052) roll requested 2024-12-09T08:18:05,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285060 2024-12-09T08:18:05,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285060 2024-12-09T08:18:05,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285060) roll requested 2024-12-09T08:18:05,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285068 2024-12-09T08:18:05,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285060 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285068 2024-12-09T08:18:05,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285068) roll requested 2024-12-09T08:18:05,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285076 2024-12-09T08:18:05,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285076 2024-12-09T08:18:05,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285076) roll requested 2024-12-09T08:18:05,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285085 2024-12-09T08:18:05,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285085 2024-12-09T08:18:05,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,093 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285085) roll requested 2024-12-09T08:18:05,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285093 2024-12-09T08:18:05,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285093 2024-12-09T08:18:05,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285093) roll requested 2024-12-09T08:18:05,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285100 2024-12-09T08:18:05,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285100 2024-12-09T08:18:05,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285100) roll requested 2024-12-09T08:18:05,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285109 2024-12-09T08:18:05,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285109 2024-12-09T08:18:05,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,118 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285109) roll requested 2024-12-09T08:18:05,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285118 2024-12-09T08:18:05,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285118 2024-12-09T08:18:05,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285118) roll requested 2024-12-09T08:18:05,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285128 2024-12-09T08:18:05,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285118 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285128 2024-12-09T08:18:05,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,135 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285128) roll requested 2024-12-09T08:18:05,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285135 2024-12-09T08:18:05,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285135 2024-12-09T08:18:05,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285135) roll requested 2024-12-09T08:18:05,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285142 2024-12-09T08:18:05,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285142 2024-12-09T08:18:05,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285142) roll requested 2024-12-09T08:18:05,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285151 2024-12-09T08:18:05,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285151 2024-12-09T08:18:05,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285151) roll requested 2024-12-09T08:18:05,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285160 2024-12-09T08:18:05,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285160 2024-12-09T08:18:05,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285160) roll requested 2024-12-09T08:18:05,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285171 2024-12-09T08:18:05,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285171 2024-12-09T08:18:05,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285171) roll requested 2024-12-09T08:18:05,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285179 2024-12-09T08:18:05,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285179 2024-12-09T08:18:05,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,187 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285179) roll requested 2024-12-09T08:18:05,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285187 2024-12-09T08:18:05,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285187 2024-12-09T08:18:05,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285187) roll requested 2024-12-09T08:18:05,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285195 2024-12-09T08:18:05,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285187 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285195 2024-12-09T08:18:05,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285195) roll requested 2024-12-09T08:18:05,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285205 2024-12-09T08:18:05,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285205 2024-12-09T08:18:05,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,213 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285205) roll requested 2024-12-09T08:18:05,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285214 2024-12-09T08:18:05,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285214 2024-12-09T08:18:05,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285214) roll requested 2024-12-09T08:18:05,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285222 2024-12-09T08:18:05,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285222 2024-12-09T08:18:05,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285222) roll requested 2024-12-09T08:18:05,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285229 2024-12-09T08:18:05,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285229 2024-12-09T08:18:05,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285229) roll requested 2024-12-09T08:18:05,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285238 2024-12-09T08:18:05,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285238 2024-12-09T08:18:05,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285238) roll requested 2024-12-09T08:18:05,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285245 2024-12-09T08:18:05,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285245 2024-12-09T08:18:05,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285245) roll requested 2024-12-09T08:18:05,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285253 2024-12-09T08:18:05,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285253 2024-12-09T08:18:05,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,260 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285253) roll requested 2024-12-09T08:18:05,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285260 2024-12-09T08:18:05,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285260 2024-12-09T08:18:05,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285260) roll requested 2024-12-09T08:18:05,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285267 2024-12-09T08:18:05,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285260 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285267 2024-12-09T08:18:05,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285267) roll requested 2024-12-09T08:18:05,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285273 2024-12-09T08:18:05,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285273 2024-12-09T08:18:05,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285273) roll requested 2024-12-09T08:18:05,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285281 2024-12-09T08:18:05,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285281 2024-12-09T08:18:05,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285281) roll requested 2024-12-09T08:18:05,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285289 2024-12-09T08:18:05,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285289 2024-12-09T08:18:05,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,298 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285289) roll requested 2024-12-09T08:18:05,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285298 2024-12-09T08:18:05,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285298 2024-12-09T08:18:05,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,307 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285298) roll requested 2024-12-09T08:18:05,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285307 2024-12-09T08:18:05,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285307 2024-12-09T08:18:05,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,316 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285307) roll requested 2024-12-09T08:18:05,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285316 2024-12-09T08:18:05,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285316 2024-12-09T08:18:05,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285316) roll requested 2024-12-09T08:18:05,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285325 2024-12-09T08:18:05,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285316 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285325 2024-12-09T08:18:05,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285325) roll requested 2024-12-09T08:18:05,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285332 2024-12-09T08:18:05,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285332 2024-12-09T08:18:05,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,341 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285332) roll requested 2024-12-09T08:18:05,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285341 2024-12-09T08:18:05,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285341 2024-12-09T08:18:05,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285341) roll requested 2024-12-09T08:18:05,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285350 2024-12-09T08:18:05,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285350 2024-12-09T08:18:05,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,358 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285350) roll requested 2024-12-09T08:18:05,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285359 2024-12-09T08:18:05,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285359 2024-12-09T08:18:05,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285359) roll requested 2024-12-09T08:18:05,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285368 2024-12-09T08:18:05,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285368 2024-12-09T08:18:05,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285368) roll requested 2024-12-09T08:18:05,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285378 2024-12-09T08:18:05,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285378 2024-12-09T08:18:05,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285378) roll requested 2024-12-09T08:18:05,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285387 2024-12-09T08:18:05,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285387 2024-12-09T08:18:05,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,397 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285387) roll requested 2024-12-09T08:18:05,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285397 2024-12-09T08:18:05,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285397 2024-12-09T08:18:05,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285397) roll requested 2024-12-09T08:18:05,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285405 2024-12-09T08:18:05,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285405 2024-12-09T08:18:05,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285405) roll requested 2024-12-09T08:18:05,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285414 2024-12-09T08:18:05,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285405 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285414 2024-12-09T08:18:05,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,423 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285414) roll requested 2024-12-09T08:18:05,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285423 2024-12-09T08:18:05,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285423 2024-12-09T08:18:05,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,430 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285423) roll requested 2024-12-09T08:18:05,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285430 2024-12-09T08:18:05,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285423 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285430 2024-12-09T08:18:05,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,438 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285430) roll requested 2024-12-09T08:18:05,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285438 2024-12-09T08:18:05,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285438 2024-12-09T08:18:05,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,448 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285438) roll requested 2024-12-09T08:18:05,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285448 2024-12-09T08:18:05,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285438 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285448 2024-12-09T08:18:05,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285448) roll requested 2024-12-09T08:18:05,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285453 2024-12-09T08:18:05,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285448 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285453 2024-12-09T08:18:05,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285453) roll requested 2024-12-09T08:18:05,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285457 2024-12-09T08:18:05,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285457 2024-12-09T08:18:05,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,463 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285457) roll requested 2024-12-09T08:18:05,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285463 2024-12-09T08:18:05,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285463 2024-12-09T08:18:05,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,470 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285463) roll requested 2024-12-09T08:18:05,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285471 2024-12-09T08:18:05,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285463 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285471 2024-12-09T08:18:05,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,477 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285471) roll requested 2024-12-09T08:18:05,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285478 2024-12-09T08:18:05,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285478 2024-12-09T08:18:05,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285478) roll requested 2024-12-09T08:18:05,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285486 2024-12-09T08:18:05,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285486 2024-12-09T08:18:05,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,494 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285486) roll requested 2024-12-09T08:18:05,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285494 2024-12-09T08:18:05,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285486 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285494 2024-12-09T08:18:05,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,503 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285494) roll requested 2024-12-09T08:18:05,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285503 2024-12-09T08:18:05,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285494 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285503 2024-12-09T08:18:05,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,510 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285503) roll requested 2024-12-09T08:18:05,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285510 2024-12-09T08:18:05,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285510 2024-12-09T08:18:05,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,518 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285510) roll requested 2024-12-09T08:18:05,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285518 2024-12-09T08:18:05,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285510 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285518 2024-12-09T08:18:05,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285518) roll requested 2024-12-09T08:18:05,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285528 2024-12-09T08:18:05,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285518 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285528 2024-12-09T08:18:05,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285528) roll requested 2024-12-09T08:18:05,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285535 2024-12-09T08:18:05,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285535 2024-12-09T08:18:05,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,542 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285535) roll requested 2024-12-09T08:18:05,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285543 2024-12-09T08:18:05,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285543 2024-12-09T08:18:05,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,550 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285543) roll requested 2024-12-09T08:18:05,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285551 2024-12-09T08:18:05,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285551 2024-12-09T08:18:05,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285551) roll requested 2024-12-09T08:18:05,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285560 2024-12-09T08:18:05,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285551 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285560 2024-12-09T08:18:05,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285560) roll requested 2024-12-09T08:18:05,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285570 2024-12-09T08:18:05,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285570 2024-12-09T08:18:05,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285570) roll requested 2024-12-09T08:18:05,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285576 2024-12-09T08:18:05,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285570 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285576 2024-12-09T08:18:05,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,581 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285576) roll requested 2024-12-09T08:18:05,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285582 2024-12-09T08:18:05,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285582 2024-12-09T08:18:05,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285582) roll requested 2024-12-09T08:18:05,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285589 2024-12-09T08:18:05,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285582 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285589 2024-12-09T08:18:05,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,597 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285589) roll requested 2024-12-09T08:18:05,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285597 2024-12-09T08:18:05,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285597 2024-12-09T08:18:05,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,606 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285597) roll requested 2024-12-09T08:18:05,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285606 2024-12-09T08:18:05,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285606 2024-12-09T08:18:05,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285606) roll requested 2024-12-09T08:18:05,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285613 2024-12-09T08:18:05,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285606 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285613 2024-12-09T08:18:05,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285613) roll requested 2024-12-09T08:18:05,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285620 2024-12-09T08:18:05,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285613 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285620 2024-12-09T08:18:05,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285620) roll requested 2024-12-09T08:18:05,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285628 2024-12-09T08:18:05,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285628 2024-12-09T08:18:05,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,638 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285628) roll requested 2024-12-09T08:18:05,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285638 2024-12-09T08:18:05,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285638 2024-12-09T08:18:05,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,647 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285638) roll requested 2024-12-09T08:18:05,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285647 2024-12-09T08:18:05,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285647 2024-12-09T08:18:05,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285647) roll requested 2024-12-09T08:18:05,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285656 2024-12-09T08:18:05,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285656 2024-12-09T08:18:05,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285656) roll requested 2024-12-09T08:18:05,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285665 2024-12-09T08:18:05,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285665 2024-12-09T08:18:05,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285665) roll requested 2024-12-09T08:18:05,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285670 2024-12-09T08:18:05,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285665 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285670 2024-12-09T08:18:05,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285670) roll requested 2024-12-09T08:18:05,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285674 2024-12-09T08:18:05,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285670 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285674 2024-12-09T08:18:05,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,681 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285674) roll requested 2024-12-09T08:18:05,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285681 2024-12-09T08:18:05,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285674 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285681 2024-12-09T08:18:05,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285681) roll requested 2024-12-09T08:18:05,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285688 2024-12-09T08:18:05,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285688 2024-12-09T08:18:05,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,695 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285688) roll requested 2024-12-09T08:18:05,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285696 2024-12-09T08:18:05,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285688 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285696 2024-12-09T08:18:05,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,703 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285696) roll requested 2024-12-09T08:18:05,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285704 2024-12-09T08:18:05,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285696 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285704 2024-12-09T08:18:05,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285704) roll requested 2024-12-09T08:18:05,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285711 2024-12-09T08:18:05,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285711 2024-12-09T08:18:05,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285711) roll requested 2024-12-09T08:18:05,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285718 2024-12-09T08:18:05,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285718 2024-12-09T08:18:05,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285718) roll requested 2024-12-09T08:18:05,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285727 2024-12-09T08:18:05,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285718 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285727 2024-12-09T08:18:05,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285727) roll requested 2024-12-09T08:18:05,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285734 2024-12-09T08:18:05,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285734 2024-12-09T08:18:05,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,740 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285734) roll requested 2024-12-09T08:18:05,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285740 2024-12-09T08:18:05,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285740 2024-12-09T08:18:05,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285740) roll requested 2024-12-09T08:18:05,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285747 2024-12-09T08:18:05,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285740 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285747 2024-12-09T08:18:05,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,755 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285747) roll requested 2024-12-09T08:18:05,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285755 2024-12-09T08:18:05,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285747 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285755 2024-12-09T08:18:05,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285755) roll requested 2024-12-09T08:18:05,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285764 2024-12-09T08:18:05,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285755 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285764 2024-12-09T08:18:05,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,771 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285764) roll requested 2024-12-09T08:18:05,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285772 2024-12-09T08:18:05,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285772 2024-12-09T08:18:05,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285772) roll requested 2024-12-09T08:18:05,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285778 2024-12-09T08:18:05,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285778 2024-12-09T08:18:05,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285778) roll requested 2024-12-09T08:18:05,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285787 2024-12-09T08:18:05,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285787 2024-12-09T08:18:05,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285787) roll requested 2024-12-09T08:18:05,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285795 2024-12-09T08:18:05,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285787 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285795 2024-12-09T08:18:05,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285795) roll requested 2024-12-09T08:18:05,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285805 2024-12-09T08:18:05,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285805 2024-12-09T08:18:05,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,814 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285805) roll requested 2024-12-09T08:18:05,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285814 2024-12-09T08:18:05,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285814 2024-12-09T08:18:05,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,824 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285814) roll requested 2024-12-09T08:18:05,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285824 2024-12-09T08:18:05,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285824 2024-12-09T08:18:05,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,834 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285824) roll requested 2024-12-09T08:18:05,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285834 2024-12-09T08:18:05,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285824 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285834 2024-12-09T08:18:05,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285834) roll requested 2024-12-09T08:18:05,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285843 2024-12-09T08:18:05,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285843 2024-12-09T08:18:05,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,851 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285843) roll requested 2024-12-09T08:18:05,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285852 2024-12-09T08:18:05,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285843 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285852 2024-12-09T08:18:05,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285852) roll requested 2024-12-09T08:18:05,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285861 2024-12-09T08:18:05,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285861 2024-12-09T08:18:05,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,869 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285861) roll requested 2024-12-09T08:18:05,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285870 2024-12-09T08:18:05,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285861 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285870 2024-12-09T08:18:05,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285870) roll requested 2024-12-09T08:18:05,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285876 2024-12-09T08:18:05,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285870 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285876 2024-12-09T08:18:05,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285876) roll requested 2024-12-09T08:18:05,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285883 2024-12-09T08:18:05,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285883 2024-12-09T08:18:05,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285883) roll requested 2024-12-09T08:18:05,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285890 2024-12-09T08:18:05,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285883 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285890 2024-12-09T08:18:05,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,898 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285890) roll requested 2024-12-09T08:18:05,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285898 2024-12-09T08:18:05,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285890 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285898 2024-12-09T08:18:05,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,905 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285898) roll requested 2024-12-09T08:18:05,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285905 2024-12-09T08:18:05,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285898 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285905 2024-12-09T08:18:05,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,911 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285905) roll requested 2024-12-09T08:18:05,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285911 2024-12-09T08:18:05,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285911 2024-12-09T08:18:05,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,918 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285911) roll requested 2024-12-09T08:18:05,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285918 2024-12-09T08:18:05,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285918 2024-12-09T08:18:05,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285918) roll requested 2024-12-09T08:18:05,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285926 2024-12-09T08:18:05,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285918 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285926 2024-12-09T08:18:05,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,933 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285926) roll requested 2024-12-09T08:18:05,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285934 2024-12-09T08:18:05,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285926 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285934 2024-12-09T08:18:05,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,942 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285934) roll requested 2024-12-09T08:18:05,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285942 2024-12-09T08:18:05,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285942 2024-12-09T08:18:05,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,949 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285942) roll requested 2024-12-09T08:18:05,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285949 2024-12-09T08:18:05,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285949 2024-12-09T08:18:05,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,956 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285949) roll requested 2024-12-09T08:18:05,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285956 2024-12-09T08:18:05,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285956 2024-12-09T08:18:05,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285956) roll requested 2024-12-09T08:18:05,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285964 2024-12-09T08:18:05,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285956 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285964 2024-12-09T08:18:05,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285964) roll requested 2024-12-09T08:18:05,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285973 2024-12-09T08:18:05,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285973 2024-12-09T08:18:05,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285973) roll requested 2024-12-09T08:18:05,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285980 2024-12-09T08:18:05,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285980 2024-12-09T08:18:05,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285980) roll requested 2024-12-09T08:18:05,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285985 2024-12-09T08:18:05,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285985 2024-12-09T08:18:05,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,990 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285985) roll requested 2024-12-09T08:18:05,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285991 2024-12-09T08:18:05,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:05,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285991 2024-12-09T08:18:05,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:05,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:05,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285991) roll requested 2024-12-09T08:18:05,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732285998 2024-12-09T08:18:06,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285998 2024-12-09T08:18:06,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732285998) roll requested 2024-12-09T08:18:06,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286005 2024-12-09T08:18:06,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732285998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286005 2024-12-09T08:18:06,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286005) roll requested 2024-12-09T08:18:06,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286014 2024-12-09T08:18:06,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286005 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286014 2024-12-09T08:18:06,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,024 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286014) roll requested 2024-12-09T08:18:06,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286024 2024-12-09T08:18:06,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286024 2024-12-09T08:18:06,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286024) roll requested 2024-12-09T08:18:06,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286033 2024-12-09T08:18:06,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286024 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286033 2024-12-09T08:18:06,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286033) roll requested 2024-12-09T08:18:06,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286041 2024-12-09T08:18:06,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286041 2024-12-09T08:18:06,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286041) roll requested 2024-12-09T08:18:06,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286047 2024-12-09T08:18:06,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286047 2024-12-09T08:18:06,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286047) roll requested 2024-12-09T08:18:06,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286055 2024-12-09T08:18:06,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286055 2024-12-09T08:18:06,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,063 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286055) roll requested 2024-12-09T08:18:06,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286064 2024-12-09T08:18:06,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286064 2024-12-09T08:18:06,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,071 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286064) roll requested 2024-12-09T08:18:06,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286071 2024-12-09T08:18:06,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286071 2024-12-09T08:18:06,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286071) roll requested 2024-12-09T08:18:06,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286078 2024-12-09T08:18:06,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286071 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286078 2024-12-09T08:18:06,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286078) roll requested 2024-12-09T08:18:06,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286086 2024-12-09T08:18:06,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286078 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286086 2024-12-09T08:18:06,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286086) roll requested 2024-12-09T08:18:06,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286093 2024-12-09T08:18:06,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286086 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286093 2024-12-09T08:18:06,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286093) roll requested 2024-12-09T08:18:06,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286100 2024-12-09T08:18:06,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286100 2024-12-09T08:18:06,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286100) roll requested 2024-12-09T08:18:06,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286108 2024-12-09T08:18:06,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286108 2024-12-09T08:18:06,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286108) roll requested 2024-12-09T08:18:06,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286116 2024-12-09T08:18:06,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286116 2024-12-09T08:18:06,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286116) roll requested 2024-12-09T08:18:06,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286122 2024-12-09T08:18:06,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286122 2024-12-09T08:18:06,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286122) roll requested 2024-12-09T08:18:06,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286129 2024-12-09T08:18:06,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286122 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286129 2024-12-09T08:18:06,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286129) roll requested 2024-12-09T08:18:06,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286137 2024-12-09T08:18:06,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286137 2024-12-09T08:18:06,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,144 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286137) roll requested 2024-12-09T08:18:06,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286144 2024-12-09T08:18:06,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286144 2024-12-09T08:18:06,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286144) roll requested 2024-12-09T08:18:06,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286151 2024-12-09T08:18:06,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286144 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286151 2024-12-09T08:18:06,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286151) roll requested 2024-12-09T08:18:06,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286158 2024-12-09T08:18:06,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286158 2024-12-09T08:18:06,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286158) roll requested 2024-12-09T08:18:06,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286164 2024-12-09T08:18:06,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286164 2024-12-09T08:18:06,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286164) roll requested 2024-12-09T08:18:06,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286171 2024-12-09T08:18:06,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286164 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286171 2024-12-09T08:18:06,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,178 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286171) roll requested 2024-12-09T08:18:06,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286178 2024-12-09T08:18:06,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286178 2024-12-09T08:18:06,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286178) roll requested 2024-12-09T08:18:06,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286190 2024-12-09T08:18:06,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286178 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286190 2024-12-09T08:18:06,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286190) roll requested 2024-12-09T08:18:06,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286196 2024-12-09T08:18:06,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286196 2024-12-09T08:18:06,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,200 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286196) roll requested 2024-12-09T08:18:06,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286200 2024-12-09T08:18:06,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286200 2024-12-09T08:18:06,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,207 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286200) roll requested 2024-12-09T08:18:06,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286207 2024-12-09T08:18:06,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286200 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286207 2024-12-09T08:18:06,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286207) roll requested 2024-12-09T08:18:06,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286215 2024-12-09T08:18:06,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286207 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286215 2024-12-09T08:18:06,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286215) roll requested 2024-12-09T08:18:06,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286222 2024-12-09T08:18:06,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286222 2024-12-09T08:18:06,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286222) roll requested 2024-12-09T08:18:06,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286231 2024-12-09T08:18:06,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286231 2024-12-09T08:18:06,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286231) roll requested 2024-12-09T08:18:06,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286239 2024-12-09T08:18:06,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286239 2024-12-09T08:18:06,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286239) roll requested 2024-12-09T08:18:06,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286246 2024-12-09T08:18:06,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286239 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286246 2024-12-09T08:18:06,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286246) roll requested 2024-12-09T08:18:06,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286252 2024-12-09T08:18:06,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286252 2024-12-09T08:18:06,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286252) roll requested 2024-12-09T08:18:06,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286260 2024-12-09T08:18:06,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286260 2024-12-09T08:18:06,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286260) roll requested 2024-12-09T08:18:06,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286268 2024-12-09T08:18:06,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286260 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286268 2024-12-09T08:18:06,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,276 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286268) roll requested 2024-12-09T08:18:06,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286277 2024-12-09T08:18:06,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286277 2024-12-09T08:18:06,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286277) roll requested 2024-12-09T08:18:06,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286286 2024-12-09T08:18:06,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286286 2024-12-09T08:18:06,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286286) roll requested 2024-12-09T08:18:06,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286294 2024-12-09T08:18:06,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286294 2024-12-09T08:18:06,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286294) roll requested 2024-12-09T08:18:06,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286299 2024-12-09T08:18:06,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286299 2024-12-09T08:18:06,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286299) roll requested 2024-12-09T08:18:06,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286305 2024-12-09T08:18:06,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286305 2024-12-09T08:18:06,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286305) roll requested 2024-12-09T08:18:06,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286313 2024-12-09T08:18:06,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286313 2024-12-09T08:18:06,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286313) roll requested 2024-12-09T08:18:06,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286322 2024-12-09T08:18:06,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286322 2024-12-09T08:18:06,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,329 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286322) roll requested 2024-12-09T08:18:06,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286329 2024-12-09T08:18:06,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286322 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286329 2024-12-09T08:18:06,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,335 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286329) roll requested 2024-12-09T08:18:06,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286336 2024-12-09T08:18:06,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286329 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286336 2024-12-09T08:18:06,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286336) roll requested 2024-12-09T08:18:06,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286344 2024-12-09T08:18:06,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286344 2024-12-09T08:18:06,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286344) roll requested 2024-12-09T08:18:06,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286354 2024-12-09T08:18:06,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286354 2024-12-09T08:18:06,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,365 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286354) roll requested 2024-12-09T08:18:06,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286365 2024-12-09T08:18:06,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286354 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286365 2024-12-09T08:18:06,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,373 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286365) roll requested 2024-12-09T08:18:06,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286373 2024-12-09T08:18:06,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286365 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286373 2024-12-09T08:18:06,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,381 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286373) roll requested 2024-12-09T08:18:06,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286381 2024-12-09T08:18:06,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286373 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286381 2024-12-09T08:18:06,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286381) roll requested 2024-12-09T08:18:06,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286388 2024-12-09T08:18:06,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286381 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286388 2024-12-09T08:18:06,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286388) roll requested 2024-12-09T08:18:06,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286397 2024-12-09T08:18:06,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286388 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286397 2024-12-09T08:18:06,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286397) roll requested 2024-12-09T08:18:06,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286407 2024-12-09T08:18:06,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286407 2024-12-09T08:18:06,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,416 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286407) roll requested 2024-12-09T08:18:06,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286416 2024-12-09T08:18:06,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286416 2024-12-09T08:18:06,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286416) roll requested 2024-12-09T08:18:06,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286423 2024-12-09T08:18:06,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286416 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286423 2024-12-09T08:18:06,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,430 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286423) roll requested 2024-12-09T08:18:06,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286430 2024-12-09T08:18:06,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286423 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286430 2024-12-09T08:18:06,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286430) roll requested 2024-12-09T08:18:06,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286437 2024-12-09T08:18:06,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286437 2024-12-09T08:18:06,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,444 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286437) roll requested 2024-12-09T08:18:06,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286444 2024-12-09T08:18:06,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286444 2024-12-09T08:18:06,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286444) roll requested 2024-12-09T08:18:06,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286453 2024-12-09T08:18:06,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286444 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286453 2024-12-09T08:18:06,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286453) roll requested 2024-12-09T08:18:06,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286461 2024-12-09T08:18:06,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286461 2024-12-09T08:18:06,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,470 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286461) roll requested 2024-12-09T08:18:06,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286471 2024-12-09T08:18:06,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286471 2024-12-09T08:18:06,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286471) roll requested 2024-12-09T08:18:06,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286480 2024-12-09T08:18:06,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286480 2024-12-09T08:18:06,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286480) roll requested 2024-12-09T08:18:06,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286488 2024-12-09T08:18:06,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286488 2024-12-09T08:18:06,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,494 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286488) roll requested 2024-12-09T08:18:06,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286494 2024-12-09T08:18:06,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286494 2024-12-09T08:18:06,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286494) roll requested 2024-12-09T08:18:06,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286501 2024-12-09T08:18:06,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286494 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286501 2024-12-09T08:18:06,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286501) roll requested 2024-12-09T08:18:06,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286508 2024-12-09T08:18:06,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286508 2024-12-09T08:18:06,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286508) roll requested 2024-12-09T08:18:06,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286516 2024-12-09T08:18:06,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286516 2024-12-09T08:18:06,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,523 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286516) roll requested 2024-12-09T08:18:06,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286523 2024-12-09T08:18:06,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286516 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286523 2024-12-09T08:18:06,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286523) roll requested 2024-12-09T08:18:06,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286531 2024-12-09T08:18:06,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286523 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286531 2024-12-09T08:18:06,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286531) roll requested 2024-12-09T08:18:06,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286540 2024-12-09T08:18:06,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286540 2024-12-09T08:18:06,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286540) roll requested 2024-12-09T08:18:06,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286548 2024-12-09T08:18:06,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286540 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286548 2024-12-09T08:18:06,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286548) roll requested 2024-12-09T08:18:06,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286555 2024-12-09T08:18:06,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286555 2024-12-09T08:18:06,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,562 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286555) roll requested 2024-12-09T08:18:06,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286562 2024-12-09T08:18:06,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286562 2024-12-09T08:18:06,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286562) roll requested 2024-12-09T08:18:06,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286571 2024-12-09T08:18:06,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286562 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286571 2024-12-09T08:18:06,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,579 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286571) roll requested 2024-12-09T08:18:06,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286580 2024-12-09T08:18:06,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286580 2024-12-09T08:18:06,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286580) roll requested 2024-12-09T08:18:06,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286588 2024-12-09T08:18:06,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286588 2024-12-09T08:18:06,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,597 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286588) roll requested 2024-12-09T08:18:06,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286597 2024-12-09T08:18:06,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286597 2024-12-09T08:18:06,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286597) roll requested 2024-12-09T08:18:06,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286605 2024-12-09T08:18:06,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286605 2024-12-09T08:18:06,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286605) roll requested 2024-12-09T08:18:06,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286612 2024-12-09T08:18:06,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286612 2024-12-09T08:18:06,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,621 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286612) roll requested 2024-12-09T08:18:06,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286621 2024-12-09T08:18:06,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286621 2024-12-09T08:18:06,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,630 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286621) roll requested 2024-12-09T08:18:06,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286630 2024-12-09T08:18:06,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286630 2024-12-09T08:18:06,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286630) roll requested 2024-12-09T08:18:06,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286638 2024-12-09T08:18:06,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286630 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286638 2024-12-09T08:18:06,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,647 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286638) roll requested 2024-12-09T08:18:06,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286647 2024-12-09T08:18:06,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286647 2024-12-09T08:18:06,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286647) roll requested 2024-12-09T08:18:06,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286653 2024-12-09T08:18:06,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286653 2024-12-09T08:18:06,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286653) roll requested 2024-12-09T08:18:06,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286660 2024-12-09T08:18:06,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286660 2024-12-09T08:18:06,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286660) roll requested 2024-12-09T08:18:06,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286668 2024-12-09T08:18:06,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286668 2024-12-09T08:18:06,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286668) roll requested 2024-12-09T08:18:06,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286676 2024-12-09T08:18:06,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286676 2024-12-09T08:18:06,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286676) roll requested 2024-12-09T08:18:06,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286684 2024-12-09T08:18:06,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286684 2024-12-09T08:18:06,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286684) roll requested 2024-12-09T08:18:06,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286690 2024-12-09T08:18:06,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286690 2024-12-09T08:18:06,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286690) roll requested 2024-12-09T08:18:06,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286698 2024-12-09T08:18:06,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286698 2024-12-09T08:18:06,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286698) roll requested 2024-12-09T08:18:06,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286705 2024-12-09T08:18:06,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286698 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286705 2024-12-09T08:18:06,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286705) roll requested 2024-12-09T08:18:06,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286713 2024-12-09T08:18:06,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286713 2024-12-09T08:18:06,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,722 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286713) roll requested 2024-12-09T08:18:06,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286722 2024-12-09T08:18:06,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286722 2024-12-09T08:18:06,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,730 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286722) roll requested 2024-12-09T08:18:06,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286730 2024-12-09T08:18:06,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286722 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286730 2024-12-09T08:18:06,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,737 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286730) roll requested 2024-12-09T08:18:06,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286737 2024-12-09T08:18:06,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286730 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286737 2024-12-09T08:18:06,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286737) roll requested 2024-12-09T08:18:06,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286745 2024-12-09T08:18:06,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286745 2024-12-09T08:18:06,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286745) roll requested 2024-12-09T08:18:06,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286753 2024-12-09T08:18:06,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286753 2024-12-09T08:18:06,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286753) roll requested 2024-12-09T08:18:06,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286761 2024-12-09T08:18:06,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286761 2024-12-09T08:18:06,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286761) roll requested 2024-12-09T08:18:06,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286769 2024-12-09T08:18:06,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286769 2024-12-09T08:18:06,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,774 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286769) roll requested 2024-12-09T08:18:06,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286774 2024-12-09T08:18:06,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286774 2024-12-09T08:18:06,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286774) roll requested 2024-12-09T08:18:06,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286781 2024-12-09T08:18:06,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286774 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286781 2024-12-09T08:18:06,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286781) roll requested 2024-12-09T08:18:06,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286788 2024-12-09T08:18:06,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286781 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286788 2024-12-09T08:18:06,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286788) roll requested 2024-12-09T08:18:06,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286796 2024-12-09T08:18:06,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286796 2024-12-09T08:18:06,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286796) roll requested 2024-12-09T08:18:06,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286805 2024-12-09T08:18:06,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286805 2024-12-09T08:18:06,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,814 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286805) roll requested 2024-12-09T08:18:06,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286815 2024-12-09T08:18:06,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286815 2024-12-09T08:18:06,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,823 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286815) roll requested 2024-12-09T08:18:06,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286824 2024-12-09T08:18:06,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286815 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286824 2024-12-09T08:18:06,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,830 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286824) roll requested 2024-12-09T08:18:06,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286830 2024-12-09T08:18:06,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286824 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286830 2024-12-09T08:18:06,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286830) roll requested 2024-12-09T08:18:06,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286837 2024-12-09T08:18:06,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286830 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286837 2024-12-09T08:18:06,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,844 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286837) roll requested 2024-12-09T08:18:06,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286845 2024-12-09T08:18:06,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286845 2024-12-09T08:18:06,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286845) roll requested 2024-12-09T08:18:06,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286853 2024-12-09T08:18:06,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286853 2024-12-09T08:18:06,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286853) roll requested 2024-12-09T08:18:06,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286860 2024-12-09T08:18:06,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286860 2024-12-09T08:18:06,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286860) roll requested 2024-12-09T08:18:06,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286867 2024-12-09T08:18:06,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286867 2024-12-09T08:18:06,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286867) roll requested 2024-12-09T08:18:06,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286875 2024-12-09T08:18:06,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286875 2024-12-09T08:18:06,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,884 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286875) roll requested 2024-12-09T08:18:06,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286884 2024-12-09T08:18:06,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286875 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286884 2024-12-09T08:18:06,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286884) roll requested 2024-12-09T08:18:06,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286894 2024-12-09T08:18:06,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286894 2024-12-09T08:18:06,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,903 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286894) roll requested 2024-12-09T08:18:06,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286903 2024-12-09T08:18:06,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286903 2024-12-09T08:18:06,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286903) roll requested 2024-12-09T08:18:06,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286912 2024-12-09T08:18:06,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286912 2024-12-09T08:18:06,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286912) roll requested 2024-12-09T08:18:06,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286920 2024-12-09T08:18:06,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286920 2024-12-09T08:18:06,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286920) roll requested 2024-12-09T08:18:06,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286927 2024-12-09T08:18:06,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286927 2024-12-09T08:18:06,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,933 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286927) roll requested 2024-12-09T08:18:06,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286934 2024-12-09T08:18:06,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286927 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286934 2024-12-09T08:18:06,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286934) roll requested 2024-12-09T08:18:06,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286940 2024-12-09T08:18:06,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286940 2024-12-09T08:18:06,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286940) roll requested 2024-12-09T08:18:06,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286947 2024-12-09T08:18:06,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286947 2024-12-09T08:18:06,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,955 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286947) roll requested 2024-12-09T08:18:06,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286956 2024-12-09T08:18:06,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286956 2024-12-09T08:18:06,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286956) roll requested 2024-12-09T08:18:06,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286986 2024-12-09T08:18:06,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:06,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286956 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286986 2024-12-09T08:18:06,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:06,995 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:06,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286986) roll requested 2024-12-09T08:18:06,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732286995 2024-12-09T08:18:07,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286986 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286995 2024-12-09T08:18:07,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732286995) roll requested 2024-12-09T08:18:07,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287003 2024-12-09T08:18:07,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732286995 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287003 2024-12-09T08:18:07,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287003) roll requested 2024-12-09T08:18:07,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287010 2024-12-09T08:18:07,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287003 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287010 2024-12-09T08:18:07,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,019 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287010) roll requested 2024-12-09T08:18:07,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287019 2024-12-09T08:18:07,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287019 2024-12-09T08:18:07,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,027 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287019) roll requested 2024-12-09T08:18:07,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287027 2024-12-09T08:18:07,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287019 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287027 2024-12-09T08:18:07,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287027) roll requested 2024-12-09T08:18:07,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287036 2024-12-09T08:18:07,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287027 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287036 2024-12-09T08:18:07,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,045 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287036) roll requested 2024-12-09T08:18:07,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287045 2024-12-09T08:18:07,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287036 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287045 2024-12-09T08:18:07,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,054 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287045) roll requested 2024-12-09T08:18:07,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287054 2024-12-09T08:18:07,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287045 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287054 2024-12-09T08:18:07,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,065 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287054) roll requested 2024-12-09T08:18:07,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287066 2024-12-09T08:18:07,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287066 2024-12-09T08:18:07,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,075 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287066) roll requested 2024-12-09T08:18:07,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287075 2024-12-09T08:18:07,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287066 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287075 2024-12-09T08:18:07,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287075) roll requested 2024-12-09T08:18:07,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287080 2024-12-09T08:18:07,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287075 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287080 2024-12-09T08:18:07,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287080) roll requested 2024-12-09T08:18:07,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287085 2024-12-09T08:18:07,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287085 2024-12-09T08:18:07,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287085) roll requested 2024-12-09T08:18:07,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287089 2024-12-09T08:18:07,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287089 2024-12-09T08:18:07,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,094 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287089) roll requested 2024-12-09T08:18:07,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287094 2024-12-09T08:18:07,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287094 2024-12-09T08:18:07,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287094) roll requested 2024-12-09T08:18:07,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287100 2024-12-09T08:18:07,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287094 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287100 2024-12-09T08:18:07,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287100) roll requested 2024-12-09T08:18:07,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287106 2024-12-09T08:18:07,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287106 2024-12-09T08:18:07,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287106) roll requested 2024-12-09T08:18:07,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287112 2024-12-09T08:18:07,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287112 2024-12-09T08:18:07,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287112) roll requested 2024-12-09T08:18:07,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287121 2024-12-09T08:18:07,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287121 2024-12-09T08:18:07,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287121) roll requested 2024-12-09T08:18:07,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287128 2024-12-09T08:18:07,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287128 2024-12-09T08:18:07,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287128) roll requested 2024-12-09T08:18:07,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287135 2024-12-09T08:18:07,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287135 2024-12-09T08:18:07,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287135) roll requested 2024-12-09T08:18:07,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287143 2024-12-09T08:18:07,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287143 2024-12-09T08:18:07,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287143) roll requested 2024-12-09T08:18:07,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287151 2024-12-09T08:18:07,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287151 2024-12-09T08:18:07,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287151) roll requested 2024-12-09T08:18:07,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287158 2024-12-09T08:18:07,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287158 2024-12-09T08:18:07,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,167 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287158) roll requested 2024-12-09T08:18:07,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287167 2024-12-09T08:18:07,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287167 2024-12-09T08:18:07,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287167) roll requested 2024-12-09T08:18:07,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287177 2024-12-09T08:18:07,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287167 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287177 2024-12-09T08:18:07,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,185 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287177) roll requested 2024-12-09T08:18:07,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287185 2024-12-09T08:18:07,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287185 2024-12-09T08:18:07,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,193 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287185) roll requested 2024-12-09T08:18:07,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287194 2024-12-09T08:18:07,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287194 2024-12-09T08:18:07,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,200 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287194) roll requested 2024-12-09T08:18:07,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287200 2024-12-09T08:18:07,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287194 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287200 2024-12-09T08:18:07,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287200) roll requested 2024-12-09T08:18:07,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287207 2024-12-09T08:18:07,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287200 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287207 2024-12-09T08:18:07,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287207) roll requested 2024-12-09T08:18:07,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287212 2024-12-09T08:18:07,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287207 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287212 2024-12-09T08:18:07,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287212) roll requested 2024-12-09T08:18:07,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287219 2024-12-09T08:18:07,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287219 2024-12-09T08:18:07,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,226 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287219) roll requested 2024-12-09T08:18:07,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287226 2024-12-09T08:18:07,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287226 2024-12-09T08:18:07,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,234 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287226) roll requested 2024-12-09T08:18:07,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287235 2024-12-09T08:18:07,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287235 2024-12-09T08:18:07,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287235) roll requested 2024-12-09T08:18:07,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287242 2024-12-09T08:18:07,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287235 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287242 2024-12-09T08:18:07,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287242) roll requested 2024-12-09T08:18:07,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287250 2024-12-09T08:18:07,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287250 2024-12-09T08:18:07,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,258 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287250) roll requested 2024-12-09T08:18:07,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287258 2024-12-09T08:18:07,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287250 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287258 2024-12-09T08:18:07,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287258) roll requested 2024-12-09T08:18:07,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287268 2024-12-09T08:18:07,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287258 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287268 2024-12-09T08:18:07,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,278 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287268) roll requested 2024-12-09T08:18:07,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287278 2024-12-09T08:18:07,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287278 2024-12-09T08:18:07,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287278) roll requested 2024-12-09T08:18:07,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287290 2024-12-09T08:18:07,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287290 2024-12-09T08:18:07,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,295 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287290) roll requested 2024-12-09T08:18:07,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287295 2024-12-09T08:18:07,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287290 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287295 2024-12-09T08:18:07,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287295) roll requested 2024-12-09T08:18:07,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287300 2024-12-09T08:18:07,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287295 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287300 2024-12-09T08:18:07,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287300) roll requested 2024-12-09T08:18:07,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287305 2024-12-09T08:18:07,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287305 2024-12-09T08:18:07,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,310 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287305) roll requested 2024-12-09T08:18:07,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287310 2024-12-09T08:18:07,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287310 2024-12-09T08:18:07,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287310) roll requested 2024-12-09T08:18:07,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287318 2024-12-09T08:18:07,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287318 2024-12-09T08:18:07,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287318) roll requested 2024-12-09T08:18:07,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287324 2024-12-09T08:18:07,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287324 2024-12-09T08:18:07,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,330 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287324) roll requested 2024-12-09T08:18:07,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287330 2024-12-09T08:18:07,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287330 2024-12-09T08:18:07,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,338 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287330) roll requested 2024-12-09T08:18:07,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287338 2024-12-09T08:18:07,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287330 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287338 2024-12-09T08:18:07,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287338) roll requested 2024-12-09T08:18:07,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287346 2024-12-09T08:18:07,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287338 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287346 2024-12-09T08:18:07,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287346) roll requested 2024-12-09T08:18:07,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287353 2024-12-09T08:18:07,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287346 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287353 2024-12-09T08:18:07,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:07,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287353) roll requested 2024-12-09T08:18:07,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287362 2024-12-09T08:18:07,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287362 2024-12-09T08:18:07,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,370 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287362) roll requested 2024-12-09T08:18:07,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287370 2024-12-09T08:18:07,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287362 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287370 2024-12-09T08:18:07,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,377 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287370) roll requested 2024-12-09T08:18:07,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287378 2024-12-09T08:18:07,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287370 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287378 2024-12-09T08:18:07,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,385 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287378) roll requested 2024-12-09T08:18:07,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287386 2024-12-09T08:18:07,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287378 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287386 2024-12-09T08:18:07,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,394 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287386) roll requested 2024-12-09T08:18:07,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287394 2024-12-09T08:18:07,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287386 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287394 2024-12-09T08:18:07,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,400 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287394) roll requested 2024-12-09T08:18:07,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287400 2024-12-09T08:18:07,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287394 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287400 2024-12-09T08:18:07,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,405 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287400) roll requested 2024-12-09T08:18:07,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287405 2024-12-09T08:18:07,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287400 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287405 2024-12-09T08:18:07,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,411 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287405) roll requested 2024-12-09T08:18:07,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287411 2024-12-09T08:18:07,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287405 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287411 2024-12-09T08:18:07,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,418 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287411) roll requested 2024-12-09T08:18:07,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287418 2024-12-09T08:18:07,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287411 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287418 2024-12-09T08:18:07,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,425 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287418) roll requested 2024-12-09T08:18:07,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287425 2024-12-09T08:18:07,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287418 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287425 2024-12-09T08:18:07,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,430 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287425) roll requested 2024-12-09T08:18:07,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287430 2024-12-09T08:18:07,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287425 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287430 2024-12-09T08:18:07,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,435 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287430) roll requested 2024-12-09T08:18:07,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287436 2024-12-09T08:18:07,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287430 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287436 2024-12-09T08:18:07,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,441 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287436) roll requested 2024-12-09T08:18:07,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287441 2024-12-09T08:18:07,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287436 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287441 2024-12-09T08:18:07,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,446 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287441) roll requested 2024-12-09T08:18:07,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287446 2024-12-09T08:18:07,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287441 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287446 2024-12-09T08:18:07,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,452 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287446) roll requested 2024-12-09T08:18:07,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287453 2024-12-09T08:18:07,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287446 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287453 2024-12-09T08:18:07,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,457 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287453) roll requested 2024-12-09T08:18:07,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287457 2024-12-09T08:18:07,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287453 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287457 2024-12-09T08:18:07,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,462 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287457) roll requested 2024-12-09T08:18:07,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287462 2024-12-09T08:18:07,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287457 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287462 2024-12-09T08:18:07,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,468 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287462) roll requested 2024-12-09T08:18:07,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287468 2024-12-09T08:18:07,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287462 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287468 2024-12-09T08:18:07,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,474 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287468) roll requested 2024-12-09T08:18:07,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287474 2024-12-09T08:18:07,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287468 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287474 2024-12-09T08:18:07,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,479 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287474) roll requested 2024-12-09T08:18:07,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287479 2024-12-09T08:18:07,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287474 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287479 2024-12-09T08:18:07,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,485 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287479) roll requested 2024-12-09T08:18:07,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287485 2024-12-09T08:18:07,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287479 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287485 2024-12-09T08:18:07,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,490 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287485) roll requested 2024-12-09T08:18:07,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287490 2024-12-09T08:18:07,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287485 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287490 2024-12-09T08:18:07,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,495 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287490) roll requested 2024-12-09T08:18:07,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287495 2024-12-09T08:18:07,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287490 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287495 2024-12-09T08:18:07,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,499 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287495) roll requested 2024-12-09T08:18:07,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287500 2024-12-09T08:18:07,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287495 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287500 2024-12-09T08:18:07,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,506 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287500) roll requested 2024-12-09T08:18:07,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287506 2024-12-09T08:18:07,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287500 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287506 2024-12-09T08:18:07,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,514 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287506) roll requested 2024-12-09T08:18:07,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287514 2024-12-09T08:18:07,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287506 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287514 2024-12-09T08:18:07,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,522 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287514) roll requested 2024-12-09T08:18:07,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287522 2024-12-09T08:18:07,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287514 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287522 2024-12-09T08:18:07,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,528 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287522) roll requested 2024-12-09T08:18:07,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287528 2024-12-09T08:18:07,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287522 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287528 2024-12-09T08:18:07,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,534 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287528) roll requested 2024-12-09T08:18:07,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287534 2024-12-09T08:18:07,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287528 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287534 2024-12-09T08:18:07,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,543 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287534) roll requested 2024-12-09T08:18:07,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287543 2024-12-09T08:18:07,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287534 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287543 2024-12-09T08:18:07,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,554 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287543) roll requested 2024-12-09T08:18:07,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287554 2024-12-09T08:18:07,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287543 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287554 2024-12-09T08:18:07,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,560 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287554) roll requested 2024-12-09T08:18:07,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287560 2024-12-09T08:18:07,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287554 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287560 2024-12-09T08:18:07,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,565 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287560) roll requested 2024-12-09T08:18:07,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287565 2024-12-09T08:18:07,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287560 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287565 2024-12-09T08:18:07,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,571 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287565) roll requested 2024-12-09T08:18:07,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287571 2024-12-09T08:18:07,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287565 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287571 2024-12-09T08:18:07,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,577 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287571) roll requested 2024-12-09T08:18:07,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287577 2024-12-09T08:18:07,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287571 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287577 2024-12-09T08:18:07,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,585 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287577) roll requested 2024-12-09T08:18:07,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287585 2024-12-09T08:18:07,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287577 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287585 2024-12-09T08:18:07,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,591 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287585) roll requested 2024-12-09T08:18:07,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287592 2024-12-09T08:18:07,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287585 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287592 2024-12-09T08:18:07,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,597 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287592) roll requested 2024-12-09T08:18:07,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287598 2024-12-09T08:18:07,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287592 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287598 2024-12-09T08:18:07,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,605 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287598) roll requested 2024-12-09T08:18:07,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287606 2024-12-09T08:18:07,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287598 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287606 2024-12-09T08:18:07,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,614 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287606) roll requested 2024-12-09T08:18:07,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287614 2024-12-09T08:18:07,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287606 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287614 2024-12-09T08:18:07,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,620 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287614) roll requested 2024-12-09T08:18:07,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287620 2024-12-09T08:18:07,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287614 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287620 2024-12-09T08:18:07,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,625 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287620) roll requested 2024-12-09T08:18:07,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287625 2024-12-09T08:18:07,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287620 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287625 2024-12-09T08:18:07,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,631 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287625) roll requested 2024-12-09T08:18:07,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287632 2024-12-09T08:18:07,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287625 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287632 2024-12-09T08:18:07,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,639 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287632) roll requested 2024-12-09T08:18:07,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287639 2024-12-09T08:18:07,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287632 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287639 2024-12-09T08:18:07,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,646 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287639) roll requested 2024-12-09T08:18:07,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287647 2024-12-09T08:18:07,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287639 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287647 2024-12-09T08:18:07,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,655 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287647) roll requested 2024-12-09T08:18:07,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287655 2024-12-09T08:18:07,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287647 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287655 2024-12-09T08:18:07,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,663 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287655) roll requested 2024-12-09T08:18:07,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287663 2024-12-09T08:18:07,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287655 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287663 2024-12-09T08:18:07,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,673 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287663) roll requested 2024-12-09T08:18:07,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287673 2024-12-09T08:18:07,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287663 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287673 2024-12-09T08:18:07,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,681 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287673) roll requested 2024-12-09T08:18:07,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287681 2024-12-09T08:18:07,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287673 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287681 2024-12-09T08:18:07,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,689 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287681) roll requested 2024-12-09T08:18:07,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287689 2024-12-09T08:18:07,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287681 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287689 2024-12-09T08:18:07,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,694 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287689) roll requested 2024-12-09T08:18:07,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287694 2024-12-09T08:18:07,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287689 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287694 2024-12-09T08:18:07,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,699 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287694) roll requested 2024-12-09T08:18:07,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287699 2024-12-09T08:18:07,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287694 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287699 2024-12-09T08:18:07,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,704 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287699) roll requested 2024-12-09T08:18:07,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287704 2024-12-09T08:18:07,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287699 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287704 2024-12-09T08:18:07,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,709 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287704) roll requested 2024-12-09T08:18:07,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287709 2024-12-09T08:18:07,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287704 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287709 2024-12-09T08:18:07,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,716 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287709) roll requested 2024-12-09T08:18:07,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287716 2024-12-09T08:18:07,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287716 2024-12-09T08:18:07,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,723 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287716) roll requested 2024-12-09T08:18:07,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287723 2024-12-09T08:18:07,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287716 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287723 2024-12-09T08:18:07,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,731 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,731 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=56, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,732 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287723) roll requested 2024-12-09T08:18:07,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287733 2024-12-09T08:18:07,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287723 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287733 2024-12-09T08:18:07,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,740 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=57, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,740 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,740 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287733) roll requested 2024-12-09T08:18:07,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287740 2024-12-09T08:18:07,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287733 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287740 2024-12-09T08:18:07,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,749 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=58, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,749 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,749 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287740) roll requested 2024-12-09T08:18:07,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287749 2024-12-09T08:18:07,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287740 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287749 2024-12-09T08:18:07,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,755 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=59, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,755 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,755 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287749) roll requested 2024-12-09T08:18:07,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287755 2024-12-09T08:18:07,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287749 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287755 2024-12-09T08:18:07,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,760 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=60, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,761 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,761 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287755) roll requested 2024-12-09T08:18:07,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287761 2024-12-09T08:18:07,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287755 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287761 2024-12-09T08:18:07,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,765 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=61, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,766 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,766 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287761) roll requested 2024-12-09T08:18:07,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287766 2024-12-09T08:18:07,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287761 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287766 2024-12-09T08:18:07,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,771 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=62, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,771 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,771 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287766) roll requested 2024-12-09T08:18:07,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287771 2024-12-09T08:18:07,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287766 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287771 2024-12-09T08:18:07,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,780 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=63, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,780 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,780 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287771) roll requested 2024-12-09T08:18:07,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287780 2024-12-09T08:18:07,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287771 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287780 2024-12-09T08:18:07,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,789 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=64, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,789 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,789 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287780) roll requested 2024-12-09T08:18:07,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287789 2024-12-09T08:18:07,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287780 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287789 2024-12-09T08:18:07,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,794 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=65, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,795 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,795 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287789) roll requested 2024-12-09T08:18:07,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287795 2024-12-09T08:18:07,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287789 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287795 2024-12-09T08:18:07,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,800 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=66, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,800 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,800 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287795) roll requested 2024-12-09T08:18:07,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287800 2024-12-09T08:18:07,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287795 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287800 2024-12-09T08:18:07,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,805 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=67, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,805 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,805 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287800) roll requested 2024-12-09T08:18:07,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287805 2024-12-09T08:18:07,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287800 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287805 2024-12-09T08:18:07,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,811 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=68, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,811 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,811 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287805) roll requested 2024-12-09T08:18:07,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287811 2024-12-09T08:18:07,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287805 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287811 2024-12-09T08:18:07,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,818 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=69, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,818 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,818 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287811) roll requested 2024-12-09T08:18:07,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287818 2024-12-09T08:18:07,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287811 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287818 2024-12-09T08:18:07,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,823 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=70, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,823 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,823 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287818) roll requested 2024-12-09T08:18:07,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287823 2024-12-09T08:18:07,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287818 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287823 2024-12-09T08:18:07,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,828 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=71, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,828 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,828 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287823) roll requested 2024-12-09T08:18:07,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287828 2024-12-09T08:18:07,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287823 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287828 2024-12-09T08:18:07,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,834 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=72, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,834 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,834 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287828) roll requested 2024-12-09T08:18:07,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287834 2024-12-09T08:18:07,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287828 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287834 2024-12-09T08:18:07,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,842 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=73, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,842 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,842 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287834) roll requested 2024-12-09T08:18:07,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287842 2024-12-09T08:18:07,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287834 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287842 2024-12-09T08:18:07,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,850 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=74, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,850 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,850 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287842) roll requested 2024-12-09T08:18:07,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287850 2024-12-09T08:18:07,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287842 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287850 2024-12-09T08:18:07,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,858 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=75, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,858 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,858 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287850) roll requested 2024-12-09T08:18:07,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287858 2024-12-09T08:18:07,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287850 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287858 2024-12-09T08:18:07,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,864 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=76, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,864 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,865 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287858) roll requested 2024-12-09T08:18:07,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287865 2024-12-09T08:18:07,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287858 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287865 2024-12-09T08:18:07,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,875 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=77, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,875 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,875 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287865) roll requested 2024-12-09T08:18:07,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287876 2024-12-09T08:18:07,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287865 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287876 2024-12-09T08:18:07,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,882 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=78, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,882 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,882 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287876) roll requested 2024-12-09T08:18:07,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287882 2024-12-09T08:18:07,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287876 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287882 2024-12-09T08:18:07,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,887 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=79, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,887 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,887 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287882) roll requested 2024-12-09T08:18:07,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287887 2024-12-09T08:18:07,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287882 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287887 2024-12-09T08:18:07,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,892 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=80, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,892 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,892 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287887) roll requested 2024-12-09T08:18:07,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287892 2024-12-09T08:18:07,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287887 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287892 2024-12-09T08:18:07,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,896 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=81, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,897 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,897 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287892) roll requested 2024-12-09T08:18:07,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287897 2024-12-09T08:18:07,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287892 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287897 2024-12-09T08:18:07,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,903 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=82, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,903 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,903 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287897) roll requested 2024-12-09T08:18:07,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287903 2024-12-09T08:18:07,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287897 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287903 2024-12-09T08:18:07,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,909 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=83, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,910 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,910 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287903) roll requested 2024-12-09T08:18:07,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287910 2024-12-09T08:18:07,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287903 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287910 2024-12-09T08:18:07,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,917 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=84, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,917 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,917 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287910) roll requested 2024-12-09T08:18:07,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287917 2024-12-09T08:18:07,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287910 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287917 2024-12-09T08:18:07,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,923 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=85, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,924 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,924 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287917) roll requested 2024-12-09T08:18:07,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287924 2024-12-09T08:18:07,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287917 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287924 2024-12-09T08:18:07,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,929 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=86, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,929 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,929 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287924) roll requested 2024-12-09T08:18:07,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287929 2024-12-09T08:18:07,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287924 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287929 2024-12-09T08:18:07,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,936 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=87, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,936 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,936 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287929) roll requested 2024-12-09T08:18:07,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287936 2024-12-09T08:18:07,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287929 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287936 2024-12-09T08:18:07,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,942 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=88, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,942 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,942 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287936) roll requested 2024-12-09T08:18:07,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287942 2024-12-09T08:18:07,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287936 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287942 2024-12-09T08:18:07,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,947 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=89, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,947 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,947 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287942) roll requested 2024-12-09T08:18:07,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287948 2024-12-09T08:18:07,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287942 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287948 2024-12-09T08:18:07,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,952 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=90, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,952 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,953 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287948) roll requested 2024-12-09T08:18:07,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287953 2024-12-09T08:18:07,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287948 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287953 2024-12-09T08:18:07,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,958 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=91, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,958 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,958 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287953) roll requested 2024-12-09T08:18:07,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287958 2024-12-09T08:18:07,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287953 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287958 2024-12-09T08:18:07,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,964 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=92, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,964 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,964 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287958) roll requested 2024-12-09T08:18:07,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287964 2024-12-09T08:18:07,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287958 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287964 2024-12-09T08:18:07,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,969 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=93, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,969 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,969 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287964) roll requested 2024-12-09T08:18:07,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287969 2024-12-09T08:18:07,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287964 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287969 2024-12-09T08:18:07,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,975 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=94, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,975 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,975 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287969) roll requested 2024-12-09T08:18:07,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287975 2024-12-09T08:18:07,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287969 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287975 2024-12-09T08:18:07,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,980 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=95, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,981 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,981 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287975) roll requested 2024-12-09T08:18:07,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287981 2024-12-09T08:18:07,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287975 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287981 2024-12-09T08:18:07,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,988 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=96, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,988 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,988 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287981) roll requested 2024-12-09T08:18:07,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287988 2024-12-09T08:18:07,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:07,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287981 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287988 2024-12-09T08:18:07,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:07,996 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=97, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:07,996 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:07,996 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:07,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287988) roll requested 2024-12-09T08:18:07,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732287996 2024-12-09T08:18:08,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287988 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287996 2024-12-09T08:18:08,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,004 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=98, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,004 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,004 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732287996) roll requested 2024-12-09T08:18:08,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288005 2024-12-09T08:18:08,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732287996 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288005 2024-12-09T08:18:08,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,011 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=99, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,011 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,011 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288005) roll requested 2024-12-09T08:18:08,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288012 2024-12-09T08:18:08,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288005 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288012 2024-12-09T08:18:08,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,019 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=100, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,019 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,019 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288012) roll requested 2024-12-09T08:18:08,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288019 2024-12-09T08:18:08,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288012 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288019 2024-12-09T08:18:08,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,027 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=101, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,027 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,027 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288019) roll requested 2024-12-09T08:18:08,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288027 2024-12-09T08:18:08,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288019 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288027 2024-12-09T08:18:08,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,035 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=102, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,036 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,036 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288027) roll requested 2024-12-09T08:18:08,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288036 2024-12-09T08:18:08,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288027 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288036 2024-12-09T08:18:08,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,044 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=103, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,044 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,044 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288036) roll requested 2024-12-09T08:18:08,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288044 2024-12-09T08:18:08,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288036 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288044 2024-12-09T08:18:08,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,052 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=104, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,052 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,052 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288044) roll requested 2024-12-09T08:18:08,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288052 2024-12-09T08:18:08,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288044 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288052 2024-12-09T08:18:08,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,059 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=105, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,059 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,059 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288052) roll requested 2024-12-09T08:18:08,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288059 2024-12-09T08:18:08,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288052 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288059 2024-12-09T08:18:08,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,068 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=106, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,068 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,068 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288059) roll requested 2024-12-09T08:18:08,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288068 2024-12-09T08:18:08,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288059 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288068 2024-12-09T08:18:08,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,076 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=107, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,076 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,076 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288068) roll requested 2024-12-09T08:18:08,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288076 2024-12-09T08:18:08,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288068 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288076 2024-12-09T08:18:08,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,083 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=108, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,084 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,084 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288076) roll requested 2024-12-09T08:18:08,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288084 2024-12-09T08:18:08,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288076 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288084 2024-12-09T08:18:08,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,094 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=109, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,094 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,094 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288084) roll requested 2024-12-09T08:18:08,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288094 2024-12-09T08:18:08,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288084 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288094 2024-12-09T08:18:08,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,101 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=110, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,101 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,101 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288094) roll requested 2024-12-09T08:18:08,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288101 2024-12-09T08:18:08,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288094 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288101 2024-12-09T08:18:08,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,107 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=111, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,107 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,107 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288101) roll requested 2024-12-09T08:18:08,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288107 2024-12-09T08:18:08,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288101 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288107 2024-12-09T08:18:08,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,113 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=112, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,113 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,113 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288107) roll requested 2024-12-09T08:18:08,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288113 2024-12-09T08:18:08,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288107 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288113 2024-12-09T08:18:08,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,120 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=113, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,120 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,120 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288113) roll requested 2024-12-09T08:18:08,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288120 2024-12-09T08:18:08,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288113 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288120 2024-12-09T08:18:08,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,127 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=114, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,128 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,128 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288120) roll requested 2024-12-09T08:18:08,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288128 2024-12-09T08:18:08,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288120 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288128 2024-12-09T08:18:08,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,135 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=115, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,135 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,135 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288128) roll requested 2024-12-09T08:18:08,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288135 2024-12-09T08:18:08,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288128 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288135 2024-12-09T08:18:08,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,141 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=116, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,141 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,141 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288135) roll requested 2024-12-09T08:18:08,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288141 2024-12-09T08:18:08,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288135 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288141 2024-12-09T08:18:08,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,148 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=117, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,148 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,148 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288141) roll requested 2024-12-09T08:18:08,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288148 2024-12-09T08:18:08,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288141 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288148 2024-12-09T08:18:08,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,155 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=118, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,155 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,156 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288148) roll requested 2024-12-09T08:18:08,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288156 2024-12-09T08:18:08,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288148 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288156 2024-12-09T08:18:08,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,164 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=119, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,164 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,165 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288156) roll requested 2024-12-09T08:18:08,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288165 2024-12-09T08:18:08,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288156 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288165 2024-12-09T08:18:08,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,174 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=120, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,174 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,174 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288165) roll requested 2024-12-09T08:18:08,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288174 2024-12-09T08:18:08,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288165 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288174 2024-12-09T08:18:08,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,182 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=121, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,182 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,182 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288174) roll requested 2024-12-09T08:18:08,182 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288182 2024-12-09T08:18:08,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288174 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288182 2024-12-09T08:18:08,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,188 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=122, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,189 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,189 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288182) roll requested 2024-12-09T08:18:08,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288189 2024-12-09T08:18:08,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288182 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288189 2024-12-09T08:18:08,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,196 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=123, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,196 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,196 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288189) roll requested 2024-12-09T08:18:08,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288196 2024-12-09T08:18:08,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288189 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288196 2024-12-09T08:18:08,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,204 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=124, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,204 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,205 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288196) roll requested 2024-12-09T08:18:08,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288205 2024-12-09T08:18:08,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288196 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288205 2024-12-09T08:18:08,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,216 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=125, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,216 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,216 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288205) roll requested 2024-12-09T08:18:08,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288216 2024-12-09T08:18:08,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288205 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288216 2024-12-09T08:18:08,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,229 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=126, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,229 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,229 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288216) roll requested 2024-12-09T08:18:08,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288230 2024-12-09T08:18:08,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288216 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288230 2024-12-09T08:18:08,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,245 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=127, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,245 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,245 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288230) roll requested 2024-12-09T08:18:08,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288245 2024-12-09T08:18:08,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288230 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288245 2024-12-09T08:18:08,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,252 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=128, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,253 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,253 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288245) roll requested 2024-12-09T08:18:08,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288253 2024-12-09T08:18:08,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288245 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288253 2024-12-09T08:18:08,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,261 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=129, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,261 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,261 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288253) roll requested 2024-12-09T08:18:08,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288261 2024-12-09T08:18:08,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288253 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288261 2024-12-09T08:18:08,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,269 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=130, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,269 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,269 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288261) roll requested 2024-12-09T08:18:08,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288269 2024-12-09T08:18:08,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288261 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288269 2024-12-09T08:18:08,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,277 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=131, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,277 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,277 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288269) roll requested 2024-12-09T08:18:08,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288277 2024-12-09T08:18:08,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288269 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288277 2024-12-09T08:18:08,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,285 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=132, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,285 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,285 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288277) roll requested 2024-12-09T08:18:08,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288285 2024-12-09T08:18:08,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288277 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288285 2024-12-09T08:18:08,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,290 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=133, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,291 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,291 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288285) roll requested 2024-12-09T08:18:08,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288291 2024-12-09T08:18:08,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288285 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288291 2024-12-09T08:18:08,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,296 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=134, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,296 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,296 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288291) roll requested 2024-12-09T08:18:08,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288297 2024-12-09T08:18:08,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288291 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288297 2024-12-09T08:18:08,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,305 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=135, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,305 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,305 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288297) roll requested 2024-12-09T08:18:08,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288305 2024-12-09T08:18:08,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288297 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288305 2024-12-09T08:18:08,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,312 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=136, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,312 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,312 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288305) roll requested 2024-12-09T08:18:08,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288312 2024-12-09T08:18:08,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288305 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288312 2024-12-09T08:18:08,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,318 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=137, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,318 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,318 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288312) roll requested 2024-12-09T08:18:08,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288318 2024-12-09T08:18:08,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288312 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288318 2024-12-09T08:18:08,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,323 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=138, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,323 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,324 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288318) roll requested 2024-12-09T08:18:08,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288324 2024-12-09T08:18:08,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288318 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288324 2024-12-09T08:18:08,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,329 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=139, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,329 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,329 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288324) roll requested 2024-12-09T08:18:08,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288329 2024-12-09T08:18:08,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288324 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288329 2024-12-09T08:18:08,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,334 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=140, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,334 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,334 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288329) roll requested 2024-12-09T08:18:08,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288334 2024-12-09T08:18:08,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288329 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288334 2024-12-09T08:18:08,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,340 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=141, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,340 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,340 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288334) roll requested 2024-12-09T08:18:08,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288340 2024-12-09T08:18:08,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288334 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288340 2024-12-09T08:18:08,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,345 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=142, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,345 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,345 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288340) roll requested 2024-12-09T08:18:08,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288345 2024-12-09T08:18:08,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288340 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288345 2024-12-09T08:18:08,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,353 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=143, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,353 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,353 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288345) roll requested 2024-12-09T08:18:08,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288353 2024-12-09T08:18:08,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288345 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288353 2024-12-09T08:18:08,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,358 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=144, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,358 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,359 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288353) roll requested 2024-12-09T08:18:08,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288359 2024-12-09T08:18:08,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288353 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288359 2024-12-09T08:18:08,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,364 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=145, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,364 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,364 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288359) roll requested 2024-12-09T08:18:08,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288364 2024-12-09T08:18:08,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288359 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288364 2024-12-09T08:18:08,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,369 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=146, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,369 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,369 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288364) roll requested 2024-12-09T08:18:08,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288370 2024-12-09T08:18:08,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288364 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288370 2024-12-09T08:18:08,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,375 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=147, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,375 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,375 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288370) roll requested 2024-12-09T08:18:08,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288375 2024-12-09T08:18:08,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288370 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288375 2024-12-09T08:18:08,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,380 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=148, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,380 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,380 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288375) roll requested 2024-12-09T08:18:08,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288381 2024-12-09T08:18:08,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288375 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288381 2024-12-09T08:18:08,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,386 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=149, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,386 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,386 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288381) roll requested 2024-12-09T08:18:08,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288386 2024-12-09T08:18:08,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288381 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288386 2024-12-09T08:18:08,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,390 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=150, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,390 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,391 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288386) roll requested 2024-12-09T08:18:08,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288391 2024-12-09T08:18:08,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288386 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288391 2024-12-09T08:18:08,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,396 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=151, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,396 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,396 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288391) roll requested 2024-12-09T08:18:08,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288396 2024-12-09T08:18:08,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288391 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288396 2024-12-09T08:18:08,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,402 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=152, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,402 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,402 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288396) roll requested 2024-12-09T08:18:08,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288402 2024-12-09T08:18:08,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288396 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288402 2024-12-09T08:18:08,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,408 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=153, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,408 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,408 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288402) roll requested 2024-12-09T08:18:08,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288409 2024-12-09T08:18:08,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288402 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288409 2024-12-09T08:18:08,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,417 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=154, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,417 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,417 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288409) roll requested 2024-12-09T08:18:08,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288417 2024-12-09T08:18:08,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288409 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288417 2024-12-09T08:18:08,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,425 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=155, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,426 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,426 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288417) roll requested 2024-12-09T08:18:08,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288426 2024-12-09T08:18:08,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288417 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288426 2024-12-09T08:18:08,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,435 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=156, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,435 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,435 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288426) roll requested 2024-12-09T08:18:08,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288435 2024-12-09T08:18:08,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288426 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288435 2024-12-09T08:18:08,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,445 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=157, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,445 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,445 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288435) roll requested 2024-12-09T08:18:08,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288445 2024-12-09T08:18:08,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288435 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288445 2024-12-09T08:18:08,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,457 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=158, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,457 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,457 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288445) roll requested 2024-12-09T08:18:08,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288457 2024-12-09T08:18:08,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288445 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288457 2024-12-09T08:18:08,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,462 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=159, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,462 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,462 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288457) roll requested 2024-12-09T08:18:08,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288462 2024-12-09T08:18:08,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288457 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288462 2024-12-09T08:18:08,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,467 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=160, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,467 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,467 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288462) roll requested 2024-12-09T08:18:08,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288467 2024-12-09T08:18:08,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288462 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288467 2024-12-09T08:18:08,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,473 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=161, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,473 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,474 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288467) roll requested 2024-12-09T08:18:08,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288474 2024-12-09T08:18:08,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288467 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288474 2024-12-09T08:18:08,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,480 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=162, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,480 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,480 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288474) roll requested 2024-12-09T08:18:08,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288480 2024-12-09T08:18:08,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288474 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288480 2024-12-09T08:18:08,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,487 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=163, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,487 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,488 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288480) roll requested 2024-12-09T08:18:08,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288488 2024-12-09T08:18:08,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288480 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288488 2024-12-09T08:18:08,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,495 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=164, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,495 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,496 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288488) roll requested 2024-12-09T08:18:08,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288496 2024-12-09T08:18:08,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288488 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288496 2024-12-09T08:18:08,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,504 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=165, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,504 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,504 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288496) roll requested 2024-12-09T08:18:08,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288504 2024-12-09T08:18:08,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288496 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288504 2024-12-09T08:18:08,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,512 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=166, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,512 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,512 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288504) roll requested 2024-12-09T08:18:08,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288512 2024-12-09T08:18:08,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288504 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288512 2024-12-09T08:18:08,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,521 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=167, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,521 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,521 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288512) roll requested 2024-12-09T08:18:08,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288521 2024-12-09T08:18:08,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288512 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288521 2024-12-09T08:18:08,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,529 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=168, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,529 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,529 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288521) roll requested 2024-12-09T08:18:08,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288529 2024-12-09T08:18:08,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288521 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288529 2024-12-09T08:18:08,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,536 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=169, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,536 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,537 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288529) roll requested 2024-12-09T08:18:08,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288537 2024-12-09T08:18:08,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288529 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288537 2024-12-09T08:18:08,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,546 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=170, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,546 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,546 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288537) roll requested 2024-12-09T08:18:08,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288546 2024-12-09T08:18:08,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288537 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288546 2024-12-09T08:18:08,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,552 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=171, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,552 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,552 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288546) roll requested 2024-12-09T08:18:08,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288552 2024-12-09T08:18:08,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288546 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288552 2024-12-09T08:18:08,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,558 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=172, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,558 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,558 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288552) roll requested 2024-12-09T08:18:08,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288559 2024-12-09T08:18:08,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288552 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288559 2024-12-09T08:18:08,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,567 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=173, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,567 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,567 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288559) roll requested 2024-12-09T08:18:08,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288567 2024-12-09T08:18:08,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288559 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288567 2024-12-09T08:18:08,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,575 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=174, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,575 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,576 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288567) roll requested 2024-12-09T08:18:08,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288576 2024-12-09T08:18:08,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288567 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288576 2024-12-09T08:18:08,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,584 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=175, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,584 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,585 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288576) roll requested 2024-12-09T08:18:08,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288585 2024-12-09T08:18:08,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288576 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288585 2024-12-09T08:18:08,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,592 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=176, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,592 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,592 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288585) roll requested 2024-12-09T08:18:08,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288592 2024-12-09T08:18:08,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288585 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288592 2024-12-09T08:18:08,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,599 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=177, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,599 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,599 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288592) roll requested 2024-12-09T08:18:08,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288599 2024-12-09T08:18:08,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288592 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288599 2024-12-09T08:18:08,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,607 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=178, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,607 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,607 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288599) roll requested 2024-12-09T08:18:08,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288607 2024-12-09T08:18:08,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288599 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288607 2024-12-09T08:18:08,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,615 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=179, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,615 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,616 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288607) roll requested 2024-12-09T08:18:08,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288616 2024-12-09T08:18:08,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288607 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288616 2024-12-09T08:18:08,622 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,622 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=180, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,622 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,623 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288616) roll requested 2024-12-09T08:18:08,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288623 2024-12-09T08:18:08,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288616 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288623 2024-12-09T08:18:08,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,628 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=181, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,628 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,628 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288623) roll requested 2024-12-09T08:18:08,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288629 2024-12-09T08:18:08,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288623 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288629 2024-12-09T08:18:08,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,635 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=182, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,635 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,635 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288629) roll requested 2024-12-09T08:18:08,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288635 2024-12-09T08:18:08,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288629 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288635 2024-12-09T08:18:08,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,642 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=183, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,642 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,643 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288635) roll requested 2024-12-09T08:18:08,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288643 2024-12-09T08:18:08,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288635 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288643 2024-12-09T08:18:08,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,651 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=184, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,651 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,651 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288643) roll requested 2024-12-09T08:18:08,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288651 2024-12-09T08:18:08,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288643 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288651 2024-12-09T08:18:08,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,660 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=185, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,660 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,660 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288651) roll requested 2024-12-09T08:18:08,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288660 2024-12-09T08:18:08,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288651 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288660 2024-12-09T08:18:08,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,672 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=186, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,672 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,672 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288660) roll requested 2024-12-09T08:18:08,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288672 2024-12-09T08:18:08,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288660 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288672 2024-12-09T08:18:08,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,678 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=187, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,678 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,678 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288672) roll requested 2024-12-09T08:18:08,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288678 2024-12-09T08:18:08,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288672 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288678 2024-12-09T08:18:08,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,683 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=188, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,683 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,683 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288678) roll requested 2024-12-09T08:18:08,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288684 2024-12-09T08:18:08,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288678 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288684 2024-12-09T08:18:08,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,688 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=189, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,688 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,688 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288684) roll requested 2024-12-09T08:18:08,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288688 2024-12-09T08:18:08,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288684 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288688 2024-12-09T08:18:08,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,694 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=190, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,694 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,694 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288688) roll requested 2024-12-09T08:18:08,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288694 2024-12-09T08:18:08,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288688 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288694 2024-12-09T08:18:08,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,700 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,700 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=191, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,700 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288694) roll requested 2024-12-09T08:18:08,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288701 2024-12-09T08:18:08,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288694 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288701 2024-12-09T08:18:08,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,708 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=192, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,708 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,708 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288701) roll requested 2024-12-09T08:18:08,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288708 2024-12-09T08:18:08,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288701 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288708 2024-12-09T08:18:08,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,717 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=193, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,717 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,717 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288708) roll requested 2024-12-09T08:18:08,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288717 2024-12-09T08:18:08,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288708 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288717 2024-12-09T08:18:08,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,723 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=194, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,723 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,724 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288717) roll requested 2024-12-09T08:18:08,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288724 2024-12-09T08:18:08,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288717 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288724 2024-12-09T08:18:08,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,730 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=195, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,730 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,731 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288724) roll requested 2024-12-09T08:18:08,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288731 2024-12-09T08:18:08,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288724 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288731 2024-12-09T08:18:08,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,739 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=196, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,739 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,739 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288731) roll requested 2024-12-09T08:18:08,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288739 2024-12-09T08:18:08,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288731 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288739 2024-12-09T08:18:08,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,748 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=197, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,748 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,749 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288739) roll requested 2024-12-09T08:18:08,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288749 2024-12-09T08:18:08,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288739 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288749 2024-12-09T08:18:08,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,756 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=198, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,756 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,757 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288749) roll requested 2024-12-09T08:18:08,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288757 2024-12-09T08:18:08,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288749 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288757 2024-12-09T08:18:08,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,763 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=199, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,763 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,763 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288757) roll requested 2024-12-09T08:18:08,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288763 2024-12-09T08:18:08,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288757 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288763 2024-12-09T08:18:08,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,768 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=200, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,768 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,768 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288763) roll requested 2024-12-09T08:18:08,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288768 2024-12-09T08:18:08,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288763 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288768 2024-12-09T08:18:08,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,773 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=201, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,773 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,773 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288768) roll requested 2024-12-09T08:18:08,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288773 2024-12-09T08:18:08,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288768 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288773 2024-12-09T08:18:08,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,783 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=202, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,783 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,783 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288773) roll requested 2024-12-09T08:18:08,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288783 2024-12-09T08:18:08,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288773 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288783 2024-12-09T08:18:08,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,789 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=203, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,790 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,790 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288783) roll requested 2024-12-09T08:18:08,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288790 2024-12-09T08:18:08,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288783 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288790 2024-12-09T08:18:08,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,800 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=204, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,800 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,801 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288790) roll requested 2024-12-09T08:18:08,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288801 2024-12-09T08:18:08,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288790 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288801 2024-12-09T08:18:08,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,806 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=205, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,806 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,806 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288801) roll requested 2024-12-09T08:18:08,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288806 2024-12-09T08:18:08,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288801 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288806 2024-12-09T08:18:08,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,811 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=206, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,811 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,811 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288806) roll requested 2024-12-09T08:18:08,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288811 2024-12-09T08:18:08,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288806 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288811 2024-12-09T08:18:08,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,817 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=207, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,817 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,817 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288811) roll requested 2024-12-09T08:18:08,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288817 2024-12-09T08:18:08,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288811 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288817 2024-12-09T08:18:08,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,823 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,823 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=208, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,823 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288817) roll requested 2024-12-09T08:18:08,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288823 2024-12-09T08:18:08,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288817 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288823 2024-12-09T08:18:08,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,828 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=209, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,828 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,828 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288823) roll requested 2024-12-09T08:18:08,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288828 2024-12-09T08:18:08,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288823 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288828 2024-12-09T08:18:08,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,834 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=210, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,834 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,834 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288828) roll requested 2024-12-09T08:18:08,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288834 2024-12-09T08:18:08,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288828 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288834 2024-12-09T08:18:08,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,839 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=211, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,839 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,839 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288834) roll requested 2024-12-09T08:18:08,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288839 2024-12-09T08:18:08,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288834 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288839 2024-12-09T08:18:08,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,847 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=212, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,847 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,847 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288839) roll requested 2024-12-09T08:18:08,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288847 2024-12-09T08:18:08,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288839 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288847 2024-12-09T08:18:08,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,854 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=213, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,854 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,854 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288847) roll requested 2024-12-09T08:18:08,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288854 2024-12-09T08:18:08,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288847 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288854 2024-12-09T08:18:08,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,866 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=214, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,866 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,866 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288854) roll requested 2024-12-09T08:18:08,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288866 2024-12-09T08:18:08,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288854 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288866 2024-12-09T08:18:08,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,871 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=215, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,871 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,871 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288866) roll requested 2024-12-09T08:18:08,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288871 2024-12-09T08:18:08,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288866 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288871 2024-12-09T08:18:08,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,876 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=216, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,876 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,876 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288871) roll requested 2024-12-09T08:18:08,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288876 2024-12-09T08:18:08,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288871 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288876 2024-12-09T08:18:08,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,881 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=217, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,881 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,881 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288876) roll requested 2024-12-09T08:18:08,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288881 2024-12-09T08:18:08,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288876 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288881 2024-12-09T08:18:08,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,888 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=218, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,887 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,888 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288881) roll requested 2024-12-09T08:18:08,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288888 2024-12-09T08:18:08,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288881 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288888 2024-12-09T08:18:08,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,895 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=219, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,895 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,895 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288888) roll requested 2024-12-09T08:18:08,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288895 2024-12-09T08:18:08,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288888 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288895 2024-12-09T08:18:08,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,903 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,903 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=220, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,903 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288895) roll requested 2024-12-09T08:18:08,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288904 2024-12-09T08:18:08,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288895 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288904 2024-12-09T08:18:08,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,912 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,912 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=221, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,912 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288904) roll requested 2024-12-09T08:18:08,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288912 2024-12-09T08:18:08,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288904 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288912 2024-12-09T08:18:08,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,923 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=222, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,923 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,923 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288912) roll requested 2024-12-09T08:18:08,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288923 2024-12-09T08:18:08,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288912 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288923 2024-12-09T08:18:08,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,933 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=223, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,933 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,933 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288923) roll requested 2024-12-09T08:18:08,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288933 2024-12-09T08:18:08,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288923 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288933 2024-12-09T08:18:08,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,939 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=224, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,939 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,939 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288933) roll requested 2024-12-09T08:18:08,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288939 2024-12-09T08:18:08,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288933 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288939 2024-12-09T08:18:08,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,944 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=225, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,944 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,945 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288939) roll requested 2024-12-09T08:18:08,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288945 2024-12-09T08:18:08,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288939 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288945 2024-12-09T08:18:08,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,950 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=226, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,950 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,950 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288945) roll requested 2024-12-09T08:18:08,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288950 2024-12-09T08:18:08,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288945 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288950 2024-12-09T08:18:08,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,955 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=227, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,955 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,955 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288950) roll requested 2024-12-09T08:18:08,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288955 2024-12-09T08:18:08,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288950 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288955 2024-12-09T08:18:08,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,962 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=228, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,962 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,962 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288955) roll requested 2024-12-09T08:18:08,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288962 2024-12-09T08:18:08,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288955 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288962 2024-12-09T08:18:08,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,970 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=229, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,969 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,970 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288962) roll requested 2024-12-09T08:18:08,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288970 2024-12-09T08:18:08,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288962 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288970 2024-12-09T08:18:08,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,977 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=230, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,977 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,978 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288970) roll requested 2024-12-09T08:18:08,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288978 2024-12-09T08:18:08,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288970 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288978 2024-12-09T08:18:08,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,984 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=231, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,984 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,984 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288978) roll requested 2024-12-09T08:18:08,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288985 2024-12-09T08:18:08,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288978 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288985 2024-12-09T08:18:08,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:08,992 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=232, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:08,992 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:08,992 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:08,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288985) roll requested 2024-12-09T08:18:08,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732288992 2024-12-09T08:18:08,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:08,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288985 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288992 2024-12-09T08:18:09,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,000 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=233, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,000 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,000 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732288992) roll requested 2024-12-09T08:18:09,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289001 2024-12-09T08:18:09,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732288992 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289001 2024-12-09T08:18:09,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,010 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=234, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,010 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,010 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289001) roll requested 2024-12-09T08:18:09,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289010 2024-12-09T08:18:09,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289001 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289010 2024-12-09T08:18:09,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,019 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=235, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,019 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,019 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289010) roll requested 2024-12-09T08:18:09,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289019 2024-12-09T08:18:09,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289010 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289019 2024-12-09T08:18:09,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,028 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=236, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,028 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,028 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289019) roll requested 2024-12-09T08:18:09,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289028 2024-12-09T08:18:09,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289019 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289028 2024-12-09T08:18:09,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,036 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,036 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=237, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,036 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289028) roll requested 2024-12-09T08:18:09,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289036 2024-12-09T08:18:09,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289028 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289036 2024-12-09T08:18:09,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,045 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=238, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,045 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,045 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289036) roll requested 2024-12-09T08:18:09,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289045 2024-12-09T08:18:09,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289036 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289045 2024-12-09T08:18:09,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,052 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=239, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,052 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,053 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289045) roll requested 2024-12-09T08:18:09,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289053 2024-12-09T08:18:09,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289045 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289053 2024-12-09T08:18:09,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,059 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=240, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,059 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,059 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289053) roll requested 2024-12-09T08:18:09,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289059 2024-12-09T08:18:09,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289053 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289059 2024-12-09T08:18:09,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,065 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,065 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=241, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,066 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289059) roll requested 2024-12-09T08:18:09,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289066 2024-12-09T08:18:09,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289059 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289066 2024-12-09T08:18:09,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,073 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,073 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=242, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,073 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289066) roll requested 2024-12-09T08:18:09,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289073 2024-12-09T08:18:09,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289066 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289073 2024-12-09T08:18:09,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,080 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,080 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=243, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,080 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289073) roll requested 2024-12-09T08:18:09,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289080 2024-12-09T08:18:09,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289073 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289080 2024-12-09T08:18:09,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,088 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=244, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,088 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,088 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289080) roll requested 2024-12-09T08:18:09,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289088 2024-12-09T08:18:09,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289080 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289088 2024-12-09T08:18:09,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,096 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,096 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=245, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,096 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289088) roll requested 2024-12-09T08:18:09,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289096 2024-12-09T08:18:09,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289088 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289096 2024-12-09T08:18:09,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,104 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,104 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=246, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,104 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289096) roll requested 2024-12-09T08:18:09,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289104 2024-12-09T08:18:09,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289096 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289104 2024-12-09T08:18:09,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,112 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,112 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=247, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,112 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289104) roll requested 2024-12-09T08:18:09,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289113 2024-12-09T08:18:09,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289104 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289113 2024-12-09T08:18:09,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,117 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=248, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,117 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,118 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289113) roll requested 2024-12-09T08:18:09,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289118 2024-12-09T08:18:09,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289113 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289118 2024-12-09T08:18:09,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,123 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,123 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=249, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,123 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289118) roll requested 2024-12-09T08:18:09,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289123 2024-12-09T08:18:09,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289118 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289123 2024-12-09T08:18:09,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,131 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=250, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,131 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,131 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289123) roll requested 2024-12-09T08:18:09,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289131 2024-12-09T08:18:09,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289123 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289131 2024-12-09T08:18:09,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,153 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=251, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,153 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,154 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289131) roll requested 2024-12-09T08:18:09,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289154 2024-12-09T08:18:09,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289131 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289154 2024-12-09T08:18:09,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,165 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,165 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=252, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,165 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289154) roll requested 2024-12-09T08:18:09,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289165 2024-12-09T08:18:09,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289154 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289165 2024-12-09T08:18:09,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,173 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,173 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=253, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,173 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289165) roll requested 2024-12-09T08:18:09,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289173 2024-12-09T08:18:09,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289165 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289173 2024-12-09T08:18:09,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,179 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=254, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,179 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,180 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289173) roll requested 2024-12-09T08:18:09,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289180 2024-12-09T08:18:09,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289173 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289180 2024-12-09T08:18:09,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,188 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,188 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=255, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,188 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289180) roll requested 2024-12-09T08:18:09,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289189 2024-12-09T08:18:09,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289180 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289189 2024-12-09T08:18:09,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,197 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=256, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,197 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,197 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289189) roll requested 2024-12-09T08:18:09,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289197 2024-12-09T08:18:09,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289189 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289197 2024-12-09T08:18:09,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,204 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,204 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=257, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,204 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289197) roll requested 2024-12-09T08:18:09,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289204 2024-12-09T08:18:09,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289197 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289204 2024-12-09T08:18:09,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,211 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,212 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=258, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,212 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289204) roll requested 2024-12-09T08:18:09,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289212 2024-12-09T08:18:09,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289204 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289212 2024-12-09T08:18:09,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,221 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=259, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,221 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,222 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289212) roll requested 2024-12-09T08:18:09,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289222 2024-12-09T08:18:09,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289212 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289222 2024-12-09T08:18:09,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,230 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=260, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,231 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,231 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289222) roll requested 2024-12-09T08:18:09,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289231 2024-12-09T08:18:09,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289222 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289231 2024-12-09T08:18:09,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,239 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=261, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,239 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,239 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289231) roll requested 2024-12-09T08:18:09,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289239 2024-12-09T08:18:09,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289231 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289239 2024-12-09T08:18:09,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,247 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,247 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=262, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,248 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289239) roll requested 2024-12-09T08:18:09,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289248 2024-12-09T08:18:09,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289239 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289248 2024-12-09T08:18:09,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,256 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,256 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=263, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,257 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289248) roll requested 2024-12-09T08:18:09,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289257 2024-12-09T08:18:09,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289248 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289257 2024-12-09T08:18:09,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,263 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,263 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=264, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,264 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289257) roll requested 2024-12-09T08:18:09,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289264 2024-12-09T08:18:09,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289257 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289264 2024-12-09T08:18:09,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,271 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=265, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,271 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,271 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289264) roll requested 2024-12-09T08:18:09,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289271 2024-12-09T08:18:09,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289264 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289271 2024-12-09T08:18:09,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,279 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,279 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=266, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,279 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289271) roll requested 2024-12-09T08:18:09,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289279 2024-12-09T08:18:09,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289271 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289279 2024-12-09T08:18:09,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,285 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=267, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,285 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,285 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289279) roll requested 2024-12-09T08:18:09,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289285 2024-12-09T08:18:09,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289279 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289285 2024-12-09T08:18:09,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,297 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,297 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=268, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,298 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289285) roll requested 2024-12-09T08:18:09,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289298 2024-12-09T08:18:09,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289285 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289298 2024-12-09T08:18:09,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,302 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=269, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,302 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,303 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289298) roll requested 2024-12-09T08:18:09,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289303 2024-12-09T08:18:09,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289298 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289303 2024-12-09T08:18:09,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,307 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=270, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,307 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,307 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289303) roll requested 2024-12-09T08:18:09,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289307 2024-12-09T08:18:09,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289303 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289307 2024-12-09T08:18:09,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,312 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,312 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=271, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,312 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289307) roll requested 2024-12-09T08:18:09,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289312 2024-12-09T08:18:09,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289307 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289312 2024-12-09T08:18:09,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,317 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=272, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,317 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,317 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289312) roll requested 2024-12-09T08:18:09,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289317 2024-12-09T08:18:09,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289312 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289317 2024-12-09T08:18:09,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,324 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,324 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=273, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,324 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289317) roll requested 2024-12-09T08:18:09,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289324 2024-12-09T08:18:09,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289317 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289324 2024-12-09T08:18:09,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,331 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,331 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=274, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,331 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289324) roll requested 2024-12-09T08:18:09,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289331 2024-12-09T08:18:09,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289324 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289331 2024-12-09T08:18:09,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,339 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,339 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=275, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,339 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289331) roll requested 2024-12-09T08:18:09,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289339 2024-12-09T08:18:09,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289331 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289339 2024-12-09T08:18:09,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,348 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,348 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=276, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,348 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289339) roll requested 2024-12-09T08:18:09,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289348 2024-12-09T08:18:09,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289339 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289348 2024-12-09T08:18:09,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,357 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,357 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=277, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,357 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289348) roll requested 2024-12-09T08:18:09,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289357 2024-12-09T08:18:09,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289348 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289357 2024-12-09T08:18:09,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,368 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,368 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=278, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,368 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289357) roll requested 2024-12-09T08:18:09,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289369 2024-12-09T08:18:09,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289357 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289369 2024-12-09T08:18:09,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,378 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=279, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,378 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,379 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289369) roll requested 2024-12-09T08:18:09,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289379 2024-12-09T08:18:09,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289369 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289379 2024-12-09T08:18:09,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,387 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,387 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=280, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,387 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289379) roll requested 2024-12-09T08:18:09,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289387 2024-12-09T08:18:09,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289379 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289387 2024-12-09T08:18:09,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,394 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=281, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,394 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,395 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289387) roll requested 2024-12-09T08:18:09,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289395 2024-12-09T08:18:09,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289387 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289395 2024-12-09T08:18:09,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,399 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,399 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=282, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,399 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289395) roll requested 2024-12-09T08:18:09,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289399 2024-12-09T08:18:09,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289395 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289399 2024-12-09T08:18:09,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,404 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,404 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=283, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,405 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289399) roll requested 2024-12-09T08:18:09,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289405 2024-12-09T08:18:09,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289399 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289405 2024-12-09T08:18:09,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,412 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,412 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=284, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,412 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289405) roll requested 2024-12-09T08:18:09,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289412 2024-12-09T08:18:09,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289405 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289412 2024-12-09T08:18:09,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,418 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,418 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=285, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,419 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289412) roll requested 2024-12-09T08:18:09,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289419 2024-12-09T08:18:09,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289412 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289419 2024-12-09T08:18:09,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,427 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,427 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=286, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,427 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289419) roll requested 2024-12-09T08:18:09,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289427 2024-12-09T08:18:09,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289419 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289427 2024-12-09T08:18:09,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,435 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=287, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,435 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,435 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289427) roll requested 2024-12-09T08:18:09,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289435 2024-12-09T08:18:09,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289427 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289435 2024-12-09T08:18:09,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,443 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,443 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=288, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,443 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289435) roll requested 2024-12-09T08:18:09,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289443 2024-12-09T08:18:09,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289435 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289443 2024-12-09T08:18:09,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,451 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,451 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=289, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,451 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289443) roll requested 2024-12-09T08:18:09,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289451 2024-12-09T08:18:09,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289443 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289451 2024-12-09T08:18:09,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,457 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,457 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=290, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,457 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289451) roll requested 2024-12-09T08:18:09,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289458 2024-12-09T08:18:09,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289451 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289458 2024-12-09T08:18:09,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,464 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,464 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=291, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,465 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289458) roll requested 2024-12-09T08:18:09,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289465 2024-12-09T08:18:09,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289458 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289465 2024-12-09T08:18:09,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,473 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,473 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=292, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,473 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289465) roll requested 2024-12-09T08:18:09,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289473 2024-12-09T08:18:09,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289465 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289473 2024-12-09T08:18:09,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,480 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,480 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=293, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,480 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289473) roll requested 2024-12-09T08:18:09,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289480 2024-12-09T08:18:09,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289473 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289480 2024-12-09T08:18:09,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,486 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,486 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=294, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,486 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289480) roll requested 2024-12-09T08:18:09,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289486 2024-12-09T08:18:09,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289480 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289486 2024-12-09T08:18:09,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,494 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,494 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=295, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,494 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289486) roll requested 2024-12-09T08:18:09,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289494 2024-12-09T08:18:09,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289486 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289494 2024-12-09T08:18:09,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,503 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,503 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=296, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,503 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289494) roll requested 2024-12-09T08:18:09,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289504 2024-12-09T08:18:09,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289494 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289504 2024-12-09T08:18:09,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,512 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,512 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=297, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,512 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289504) roll requested 2024-12-09T08:18:09,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289513 2024-12-09T08:18:09,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289504 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289513 2024-12-09T08:18:09,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,521 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=298, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,521 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,521 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289513) roll requested 2024-12-09T08:18:09,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289521 2024-12-09T08:18:09,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289513 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289521 2024-12-09T08:18:09,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,528 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=299, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,528 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,528 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289521) roll requested 2024-12-09T08:18:09,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289528 2024-12-09T08:18:09,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289521 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289528 2024-12-09T08:18:09,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,535 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,535 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=300, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,536 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289528) roll requested 2024-12-09T08:18:09,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289536 2024-12-09T08:18:09,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289528 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289536 2024-12-09T08:18:09,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,542 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,543 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=301, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,543 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289536) roll requested 2024-12-09T08:18:09,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289543 2024-12-09T08:18:09,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289536 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289543 2024-12-09T08:18:09,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,549 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=302, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,549 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,550 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289543) roll requested 2024-12-09T08:18:09,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289550 2024-12-09T08:18:09,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289543 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289550 2024-12-09T08:18:09,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,558 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,558 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=303, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,559 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289550) roll requested 2024-12-09T08:18:09,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289559 2024-12-09T08:18:09,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289550 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289559 2024-12-09T08:18:09,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,568 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,568 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=304, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,568 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289559) roll requested 2024-12-09T08:18:09,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289568 2024-12-09T08:18:09,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289559 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289568 2024-12-09T08:18:09,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,573 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,573 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=305, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,573 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289568) roll requested 2024-12-09T08:18:09,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289573 2024-12-09T08:18:09,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289568 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289573 2024-12-09T08:18:09,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,580 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,580 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=306, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,580 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289573) roll requested 2024-12-09T08:18:09,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289580 2024-12-09T08:18:09,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289573 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289580 2024-12-09T08:18:09,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,587 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,587 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=307, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,588 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289580) roll requested 2024-12-09T08:18:09,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289588 2024-12-09T08:18:09,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289580 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289588 2024-12-09T08:18:09,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,595 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,595 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=308, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,595 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289588) roll requested 2024-12-09T08:18:09,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289596 2024-12-09T08:18:09,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289588 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289596 2024-12-09T08:18:09,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,603 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,603 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=309, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,603 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289596) roll requested 2024-12-09T08:18:09,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289603 2024-12-09T08:18:09,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289596 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289603 2024-12-09T08:18:09,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,611 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,611 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=310, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,611 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289603) roll requested 2024-12-09T08:18:09,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289611 2024-12-09T08:18:09,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289603 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289611 2024-12-09T08:18:09,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,621 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,621 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=311, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,621 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289611) roll requested 2024-12-09T08:18:09,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289622 2024-12-09T08:18:09,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289611 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289622 2024-12-09T08:18:09,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,629 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,629 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=312, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,630 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289622) roll requested 2024-12-09T08:18:09,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289630 2024-12-09T08:18:09,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289622 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289630 2024-12-09T08:18:09,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,637 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=313, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,637 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,637 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289630) roll requested 2024-12-09T08:18:09,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289637 2024-12-09T08:18:09,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289630 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289637 2024-12-09T08:18:09,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,645 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,645 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=314, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,645 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289637) roll requested 2024-12-09T08:18:09,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289645 2024-12-09T08:18:09,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289637 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289645 2024-12-09T08:18:09,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,652 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,652 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=315, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,652 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289645) roll requested 2024-12-09T08:18:09,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289652 2024-12-09T08:18:09,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289645 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289652 2024-12-09T08:18:09,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,658 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,658 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=316, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,658 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289652) roll requested 2024-12-09T08:18:09,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289658 2024-12-09T08:18:09,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289652 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289658 2024-12-09T08:18:09,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,668 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,668 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=317, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,669 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289658) roll requested 2024-12-09T08:18:09,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289669 2024-12-09T08:18:09,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289658 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289669 2024-12-09T08:18:09,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,674 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,674 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=318, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,674 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289669) roll requested 2024-12-09T08:18:09,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289674 2024-12-09T08:18:09,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289669 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289674 2024-12-09T08:18:09,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,680 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=319, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,680 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,680 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289674) roll requested 2024-12-09T08:18:09,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289680 2024-12-09T08:18:09,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289674 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289680 2024-12-09T08:18:09,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,687 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=320, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,687 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,687 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289680) roll requested 2024-12-09T08:18:09,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289687 2024-12-09T08:18:09,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289680 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289687 2024-12-09T08:18:09,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,695 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=321, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,695 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,695 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289687) roll requested 2024-12-09T08:18:09,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289695 2024-12-09T08:18:09,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289687 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289695 2024-12-09T08:18:09,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,702 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=322, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,702 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,702 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289695) roll requested 2024-12-09T08:18:09,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289702 2024-12-09T08:18:09,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289695 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289702 2024-12-09T08:18:09,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,709 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,709 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=323, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,709 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289702) roll requested 2024-12-09T08:18:09,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289709 2024-12-09T08:18:09,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289702 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289709 2024-12-09T08:18:09,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,717 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=324, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,717 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,717 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289709) roll requested 2024-12-09T08:18:09,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289717 2024-12-09T08:18:09,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289717 2024-12-09T08:18:09,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,726 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,726 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=325, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,726 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289717) roll requested 2024-12-09T08:18:09,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289726 2024-12-09T08:18:09,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289717 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289726 2024-12-09T08:18:09,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,735 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=326, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,735 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,735 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289726) roll requested 2024-12-09T08:18:09,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289735 2024-12-09T08:18:09,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289726 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289735 2024-12-09T08:18:09,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,742 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,742 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=327, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,742 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289735) roll requested 2024-12-09T08:18:09,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289742 2024-12-09T08:18:09,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289735 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289742 2024-12-09T08:18:09,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,751 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,751 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=328, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,751 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289742) roll requested 2024-12-09T08:18:09,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289752 2024-12-09T08:18:09,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289742 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289752 2024-12-09T08:18:09,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,759 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,760 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=329, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,760 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289752) roll requested 2024-12-09T08:18:09,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289760 2024-12-09T08:18:09,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289752 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289760 2024-12-09T08:18:09,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,767 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,767 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=330, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,768 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289760) roll requested 2024-12-09T08:18:09,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289768 2024-12-09T08:18:09,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289760 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289768 2024-12-09T08:18:09,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,777 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,777 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=331, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,777 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289768) roll requested 2024-12-09T08:18:09,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289777 2024-12-09T08:18:09,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289768 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289777 2024-12-09T08:18:09,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,787 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,787 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=332, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,787 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289777) roll requested 2024-12-09T08:18:09,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289787 2024-12-09T08:18:09,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289777 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289787 2024-12-09T08:18:09,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,796 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,796 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=333, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,796 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289787) roll requested 2024-12-09T08:18:09,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289796 2024-12-09T08:18:09,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289787 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289796 2024-12-09T08:18:09,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,805 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=334, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,805 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,806 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289796) roll requested 2024-12-09T08:18:09,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289806 2024-12-09T08:18:09,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289796 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289806 2024-12-09T08:18:09,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,814 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,815 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=335, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,815 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289806) roll requested 2024-12-09T08:18:09,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289815 2024-12-09T08:18:09,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289806 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289815 2024-12-09T08:18:09,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,823 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=336, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,823 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,823 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289815) roll requested 2024-12-09T08:18:09,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289823 2024-12-09T08:18:09,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289815 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289823 2024-12-09T08:18:09,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,831 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,831 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=337, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,831 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289823) roll requested 2024-12-09T08:18:09,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289831 2024-12-09T08:18:09,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289823 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289831 2024-12-09T08:18:09,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,841 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=338, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,841 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,842 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289831) roll requested 2024-12-09T08:18:09,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289842 2024-12-09T08:18:09,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289831 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289842 2024-12-09T08:18:09,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,851 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,852 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=339, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,852 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289842) roll requested 2024-12-09T08:18:09,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289852 2024-12-09T08:18:09,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289842 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289852 2024-12-09T08:18:09,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,859 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,859 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=340, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,859 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289852) roll requested 2024-12-09T08:18:09,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289859 2024-12-09T08:18:09,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289852 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289859 2024-12-09T08:18:09,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,866 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,866 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=341, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,866 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289859) roll requested 2024-12-09T08:18:09,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289866 2024-12-09T08:18:09,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289859 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289866 2024-12-09T08:18:09,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,874 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,874 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=342, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,874 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289866) roll requested 2024-12-09T08:18:09,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289874 2024-12-09T08:18:09,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289866 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289874 2024-12-09T08:18:09,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,881 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,881 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=343, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,881 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289874) roll requested 2024-12-09T08:18:09,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289881 2024-12-09T08:18:09,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289874 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289881 2024-12-09T08:18:09,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,887 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,887 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=344, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,888 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289881) roll requested 2024-12-09T08:18:09,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289888 2024-12-09T08:18:09,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289881 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289888 2024-12-09T08:18:09,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,895 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,895 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=345, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,895 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289888) roll requested 2024-12-09T08:18:09,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289895 2024-12-09T08:18:09,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289888 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289895 2024-12-09T08:18:09,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,902 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,902 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=346, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,902 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289895) roll requested 2024-12-09T08:18:09,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289902 2024-12-09T08:18:09,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289895 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289902 2024-12-09T08:18:09,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,909 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=347, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,909 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,909 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289902) roll requested 2024-12-09T08:18:09,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289910 2024-12-09T08:18:09,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289902 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289910 2024-12-09T08:18:09,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,917 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,917 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=348, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,917 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289910) roll requested 2024-12-09T08:18:09,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289918 2024-12-09T08:18:09,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289910 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289918 2024-12-09T08:18:09,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,925 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=349, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,925 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,925 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289918) roll requested 2024-12-09T08:18:09,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289925 2024-12-09T08:18:09,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289918 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289925 2024-12-09T08:18:09,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,932 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,932 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=350, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,932 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289925) roll requested 2024-12-09T08:18:09,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289933 2024-12-09T08:18:09,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289925 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289933 2024-12-09T08:18:09,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,940 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,940 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=351, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,940 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289933) roll requested 2024-12-09T08:18:09,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289941 2024-12-09T08:18:09,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289933 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289941 2024-12-09T08:18:09,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,949 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=352, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,948 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,949 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289941) roll requested 2024-12-09T08:18:09,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289949 2024-12-09T08:18:09,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289941 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289949 2024-12-09T08:18:09,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,959 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,959 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=353, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,959 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289949) roll requested 2024-12-09T08:18:09,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289959 2024-12-09T08:18:09,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289949 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289959 2024-12-09T08:18:09,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,967 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,967 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=354, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,967 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289959) roll requested 2024-12-09T08:18:09,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289967 2024-12-09T08:18:09,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289959 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289967 2024-12-09T08:18:09,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,975 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,976 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=355, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,976 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289967) roll requested 2024-12-09T08:18:09,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289976 2024-12-09T08:18:09,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289967 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289976 2024-12-09T08:18:09,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,981 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,981 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=356, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,981 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289976) roll requested 2024-12-09T08:18:09,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289981 2024-12-09T08:18:09,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289976 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289981 2024-12-09T08:18:09,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,986 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,986 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=357, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,986 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289981) roll requested 2024-12-09T08:18:09,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289987 2024-12-09T08:18:09,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289981 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289987 2024-12-09T08:18:09,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,992 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,992 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=358, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,993 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289987) roll requested 2024-12-09T08:18:09,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289993 2024-12-09T08:18:09,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:09,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289987 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289993 2024-12-09T08:18:09,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:09,999 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,999 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=359, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:09,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:09,999 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:09,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289993) roll requested 2024-12-09T08:18:09,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732289999 2024-12-09T08:18:10,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289993 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289999 2024-12-09T08:18:10,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,006 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,006 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=360, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,006 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732289999) roll requested 2024-12-09T08:18:10,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290006 2024-12-09T08:18:10,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732289999 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290006 2024-12-09T08:18:10,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,015 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,015 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=361, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,015 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290006) roll requested 2024-12-09T08:18:10,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290015 2024-12-09T08:18:10,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290006 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290015 2024-12-09T08:18:10,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,023 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,023 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=362, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,023 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290015) roll requested 2024-12-09T08:18:10,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290023 2024-12-09T08:18:10,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290015 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290023 2024-12-09T08:18:10,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,030 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=363, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,030 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,030 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290023) roll requested 2024-12-09T08:18:10,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290031 2024-12-09T08:18:10,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290023 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290031 2024-12-09T08:18:10,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,044 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,044 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=364, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,044 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290031) roll requested 2024-12-09T08:18:10,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290044 2024-12-09T08:18:10,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290031 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290044 2024-12-09T08:18:10,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,050 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,050 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=365, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,050 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290044) roll requested 2024-12-09T08:18:10,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290050 2024-12-09T08:18:10,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290044 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290050 2024-12-09T08:18:10,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,055 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,055 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=366, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,056 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290050) roll requested 2024-12-09T08:18:10,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290056 2024-12-09T08:18:10,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290050 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290056 2024-12-09T08:18:10,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,061 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,061 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=367, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,061 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290056) roll requested 2024-12-09T08:18:10,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290061 2024-12-09T08:18:10,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290056 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290061 2024-12-09T08:18:10,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,068 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=368, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,068 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,068 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290061) roll requested 2024-12-09T08:18:10,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290068 2024-12-09T08:18:10,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290061 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290068 2024-12-09T08:18:10,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,075 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=369, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,075 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,076 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290068) roll requested 2024-12-09T08:18:10,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290076 2024-12-09T08:18:10,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290068 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290076 2024-12-09T08:18:10,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,081 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,081 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=370, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,081 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290076) roll requested 2024-12-09T08:18:10,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290081 2024-12-09T08:18:10,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290076 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290081 2024-12-09T08:18:10,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,087 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,087 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=371, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,087 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290081) roll requested 2024-12-09T08:18:10,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290087 2024-12-09T08:18:10,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290081 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290087 2024-12-09T08:18:10,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,094 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,094 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=372, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,094 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290087) roll requested 2024-12-09T08:18:10,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290094 2024-12-09T08:18:10,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290087 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290094 2024-12-09T08:18:10,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,101 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,101 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=373, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,102 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290094) roll requested 2024-12-09T08:18:10,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290102 2024-12-09T08:18:10,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290094 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290102 2024-12-09T08:18:10,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,108 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,108 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=374, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,108 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290102) roll requested 2024-12-09T08:18:10,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290108 2024-12-09T08:18:10,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290102 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290108 2024-12-09T08:18:10,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,115 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,115 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=375, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,115 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290108) roll requested 2024-12-09T08:18:10,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290115 2024-12-09T08:18:10,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290108 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290115 2024-12-09T08:18:10,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,123 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,123 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=376, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,124 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290115) roll requested 2024-12-09T08:18:10,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290124 2024-12-09T08:18:10,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290115 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290124 2024-12-09T08:18:10,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,131 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,131 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=377, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,132 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290124) roll requested 2024-12-09T08:18:10,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290132 2024-12-09T08:18:10,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290124 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290132 2024-12-09T08:18:10,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,141 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=378, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,141 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,141 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290132) roll requested 2024-12-09T08:18:10,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290141 2024-12-09T08:18:10,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290132 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290141 2024-12-09T08:18:10,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,149 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,149 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=379, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,149 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290141) roll requested 2024-12-09T08:18:10,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290149 2024-12-09T08:18:10,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290141 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290149 2024-12-09T08:18:10,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,157 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,157 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=380, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,157 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290149) roll requested 2024-12-09T08:18:10,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290157 2024-12-09T08:18:10,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290149 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290157 2024-12-09T08:18:10,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,165 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,165 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=381, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,165 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290157) roll requested 2024-12-09T08:18:10,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290165 2024-12-09T08:18:10,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290157 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290165 2024-12-09T08:18:10,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,172 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,172 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=382, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,172 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290165) roll requested 2024-12-09T08:18:10,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290172 2024-12-09T08:18:10,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290165 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290172 2024-12-09T08:18:10,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,179 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=383, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,179 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,180 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290172) roll requested 2024-12-09T08:18:10,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290180 2024-12-09T08:18:10,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290172 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290180 2024-12-09T08:18:10,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,188 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=384, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,188 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,189 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290180) roll requested 2024-12-09T08:18:10,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290189 2024-12-09T08:18:10,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290180 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290189 2024-12-09T08:18:10,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,196 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,196 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=385, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,196 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290189) roll requested 2024-12-09T08:18:10,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290196 2024-12-09T08:18:10,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290189 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290196 2024-12-09T08:18:10,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,202 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=386, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,202 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,203 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290196) roll requested 2024-12-09T08:18:10,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290203 2024-12-09T08:18:10,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290196 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290203 2024-12-09T08:18:10,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,211 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,211 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=387, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,211 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290203) roll requested 2024-12-09T08:18:10,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290211 2024-12-09T08:18:10,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290203 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290211 2024-12-09T08:18:10,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,219 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,219 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=388, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,219 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290211) roll requested 2024-12-09T08:18:10,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290219 2024-12-09T08:18:10,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290211 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290219 2024-12-09T08:18:10,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,227 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,227 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=389, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,227 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290219) roll requested 2024-12-09T08:18:10,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290227 2024-12-09T08:18:10,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290219 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290227 2024-12-09T08:18:10,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,236 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,237 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=390, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,237 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290227) roll requested 2024-12-09T08:18:10,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290237 2024-12-09T08:18:10,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290227 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290237 2024-12-09T08:18:10,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,247 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,247 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=391, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,248 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290237) roll requested 2024-12-09T08:18:10,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290248 2024-12-09T08:18:10,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290237 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290248 2024-12-09T08:18:10,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,255 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,255 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=392, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,255 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290248) roll requested 2024-12-09T08:18:10,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290255 2024-12-09T08:18:10,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290248 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290255 2024-12-09T08:18:10,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,262 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,262 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=393, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,262 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290255) roll requested 2024-12-09T08:18:10,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290262 2024-12-09T08:18:10,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290255 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290262 2024-12-09T08:18:10,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,269 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,269 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=394, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,269 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290262) roll requested 2024-12-09T08:18:10,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290269 2024-12-09T08:18:10,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290262 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290269 2024-12-09T08:18:10,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,277 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,277 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=395, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,277 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290269) roll requested 2024-12-09T08:18:10,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290277 2024-12-09T08:18:10,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290269 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290277 2024-12-09T08:18:10,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,285 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,285 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=396, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,285 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290277) roll requested 2024-12-09T08:18:10,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290285 2024-12-09T08:18:10,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290277 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290285 2024-12-09T08:18:10,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,293 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,293 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=397, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,293 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290285) roll requested 2024-12-09T08:18:10,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290294 2024-12-09T08:18:10,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290285 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290294 2024-12-09T08:18:10,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,303 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,303 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=398, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,304 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290294) roll requested 2024-12-09T08:18:10,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290304 2024-12-09T08:18:10,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290294 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290304 2024-12-09T08:18:10,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,312 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,312 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=399, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,312 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290304) roll requested 2024-12-09T08:18:10,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290312 2024-12-09T08:18:10,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290304 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290312 2024-12-09T08:18:10,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,320 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,321 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=400, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,321 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290312) roll requested 2024-12-09T08:18:10,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290321 2024-12-09T08:18:10,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290312 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290321 2024-12-09T08:18:10,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,328 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,328 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=401, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,328 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290321) roll requested 2024-12-09T08:18:10,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290329 2024-12-09T08:18:10,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290321 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290329 2024-12-09T08:18:10,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,334 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,334 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=402, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,334 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290329) roll requested 2024-12-09T08:18:10,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290334 2024-12-09T08:18:10,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290329 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290334 2024-12-09T08:18:10,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,340 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,340 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=403, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,340 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290334) roll requested 2024-12-09T08:18:10,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290340 2024-12-09T08:18:10,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290334 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290340 2024-12-09T08:18:10,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,348 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,348 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=404, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,348 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290340) roll requested 2024-12-09T08:18:10,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290348 2024-12-09T08:18:10,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290340 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290348 2024-12-09T08:18:10,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,356 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,356 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=405, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,356 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290348) roll requested 2024-12-09T08:18:10,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290356 2024-12-09T08:18:10,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290348 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290356 2024-12-09T08:18:10,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,364 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,364 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=406, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,364 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290356) roll requested 2024-12-09T08:18:10,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290364 2024-12-09T08:18:10,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290356 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290364 2024-12-09T08:18:10,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,372 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,372 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=407, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,372 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290364) roll requested 2024-12-09T08:18:10,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290373 2024-12-09T08:18:10,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290364 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290373 2024-12-09T08:18:10,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,380 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,380 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=408, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,380 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290373) roll requested 2024-12-09T08:18:10,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290380 2024-12-09T08:18:10,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290373 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290380 2024-12-09T08:18:10,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,389 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,389 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=409, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,390 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290380) roll requested 2024-12-09T08:18:10,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290390 2024-12-09T08:18:10,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290380 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290390 2024-12-09T08:18:10,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,400 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,400 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=410, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,400 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290390) roll requested 2024-12-09T08:18:10,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290400 2024-12-09T08:18:10,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290390 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290400 2024-12-09T08:18:10,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,407 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,407 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=411, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,408 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290400) roll requested 2024-12-09T08:18:10,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290408 2024-12-09T08:18:10,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290400 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290408 2024-12-09T08:18:10,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,416 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,416 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=412, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,416 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290408) roll requested 2024-12-09T08:18:10,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290416 2024-12-09T08:18:10,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290408 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290416 2024-12-09T08:18:10,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,425 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,425 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=413, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,426 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290416) roll requested 2024-12-09T08:18:10,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290426 2024-12-09T08:18:10,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290416 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290426 2024-12-09T08:18:10,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,432 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,432 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=414, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,432 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290426) roll requested 2024-12-09T08:18:10,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290432 2024-12-09T08:18:10,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290426 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290432 2024-12-09T08:18:10,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,439 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,439 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=415, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,439 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290432) roll requested 2024-12-09T08:18:10,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290439 2024-12-09T08:18:10,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290432 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290439 2024-12-09T08:18:10,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,446 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,446 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=416, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,447 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290439) roll requested 2024-12-09T08:18:10,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290447 2024-12-09T08:18:10,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290439 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290447 2024-12-09T08:18:10,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,455 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,455 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=417, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,455 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290447) roll requested 2024-12-09T08:18:10,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290455 2024-12-09T08:18:10,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290447 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290455 2024-12-09T08:18:10,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,464 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,464 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=418, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,465 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290455) roll requested 2024-12-09T08:18:10,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290465 2024-12-09T08:18:10,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290455 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290465 2024-12-09T08:18:10,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,473 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,473 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=419, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,473 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290465) roll requested 2024-12-09T08:18:10,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290474 2024-12-09T08:18:10,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290465 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290474 2024-12-09T08:18:10,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,479 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,479 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=420, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,479 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290474) roll requested 2024-12-09T08:18:10,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290479 2024-12-09T08:18:10,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290474 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290479 2024-12-09T08:18:10,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,486 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,486 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=421, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,486 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290479) roll requested 2024-12-09T08:18:10,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290486 2024-12-09T08:18:10,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290479 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290486 2024-12-09T08:18:10,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,493 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,493 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=422, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,493 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290486) roll requested 2024-12-09T08:18:10,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290493 2024-12-09T08:18:10,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290486 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290493 2024-12-09T08:18:10,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,502 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,502 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=423, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,503 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290493) roll requested 2024-12-09T08:18:10,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290503 2024-12-09T08:18:10,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290493 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290503 2024-12-09T08:18:10,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,509 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,509 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=424, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,509 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290503) roll requested 2024-12-09T08:18:10,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290509 2024-12-09T08:18:10,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290503 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290509 2024-12-09T08:18:10,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,515 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,515 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=425, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,515 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290509) roll requested 2024-12-09T08:18:10,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290515 2024-12-09T08:18:10,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290509 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290515 2024-12-09T08:18:10,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,522 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,522 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=426, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,522 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290515) roll requested 2024-12-09T08:18:10,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290522 2024-12-09T08:18:10,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290515 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290522 2024-12-09T08:18:10,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,529 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,529 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=427, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,529 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290522) roll requested 2024-12-09T08:18:10,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290530 2024-12-09T08:18:10,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290522 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290530 2024-12-09T08:18:10,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,538 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,538 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=428, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,538 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290530) roll requested 2024-12-09T08:18:10,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290538 2024-12-09T08:18:10,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290530 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290538 2024-12-09T08:18:10,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,545 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,545 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=429, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,545 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290538) roll requested 2024-12-09T08:18:10,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290545 2024-12-09T08:18:10,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290538 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290545 2024-12-09T08:18:10,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,552 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,552 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=430, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,552 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290545) roll requested 2024-12-09T08:18:10,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290552 2024-12-09T08:18:10,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290545 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290552 2024-12-09T08:18:10,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,559 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,559 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=431, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,559 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290552) roll requested 2024-12-09T08:18:10,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290559 2024-12-09T08:18:10,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290552 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290559 2024-12-09T08:18:10,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,567 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,567 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=432, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,568 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290559) roll requested 2024-12-09T08:18:10,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290568 2024-12-09T08:18:10,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290559 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290568 2024-12-09T08:18:10,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,577 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,577 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=433, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,577 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290568) roll requested 2024-12-09T08:18:10,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290577 2024-12-09T08:18:10,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290568 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290577 2024-12-09T08:18:10,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,589 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,589 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=434, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,589 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290577) roll requested 2024-12-09T08:18:10,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290589 2024-12-09T08:18:10,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290577 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290589 2024-12-09T08:18:10,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,595 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=435, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,595 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,596 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290589) roll requested 2024-12-09T08:18:10,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290596 2024-12-09T08:18:10,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290589 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290596 2024-12-09T08:18:10,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,600 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,600 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=436, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,601 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290596) roll requested 2024-12-09T08:18:10,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290601 2024-12-09T08:18:10,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290596 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290601 2024-12-09T08:18:10,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,606 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,606 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=437, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,606 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290601) roll requested 2024-12-09T08:18:10,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290606 2024-12-09T08:18:10,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290601 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290606 2024-12-09T08:18:10,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,612 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,612 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=438, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,612 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290606) roll requested 2024-12-09T08:18:10,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290612 2024-12-09T08:18:10,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290606 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290612 2024-12-09T08:18:10,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,618 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,619 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=439, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,619 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290612) roll requested 2024-12-09T08:18:10,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290619 2024-12-09T08:18:10,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290612 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290619 2024-12-09T08:18:10,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,629 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,629 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=440, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,629 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290619) roll requested 2024-12-09T08:18:10,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290629 2024-12-09T08:18:10,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290619 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290629 2024-12-09T08:18:10,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,638 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,639 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=441, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,639 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290629) roll requested 2024-12-09T08:18:10,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290639 2024-12-09T08:18:10,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290629 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290639 2024-12-09T08:18:10,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,645 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,645 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=442, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,645 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290639) roll requested 2024-12-09T08:18:10,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290645 2024-12-09T08:18:10,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290639 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290645 2024-12-09T08:18:10,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,649 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,649 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=443, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,650 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290645) roll requested 2024-12-09T08:18:10,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290650 2024-12-09T08:18:10,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290645 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290650 2024-12-09T08:18:10,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,655 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,655 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=444, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,655 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290650) roll requested 2024-12-09T08:18:10,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290655 2024-12-09T08:18:10,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290650 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290655 2024-12-09T08:18:10,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,661 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,661 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=445, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,662 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290655) roll requested 2024-12-09T08:18:10,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290662 2024-12-09T08:18:10,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290655 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290662 2024-12-09T08:18:10,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,668 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,669 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=446, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,669 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290662) roll requested 2024-12-09T08:18:10,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290669 2024-12-09T08:18:10,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290662 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290669 2024-12-09T08:18:10,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,676 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,676 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=447, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,677 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290669) roll requested 2024-12-09T08:18:10,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290677 2024-12-09T08:18:10,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290669 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290677 2024-12-09T08:18:10,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,683 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=448, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,683 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,683 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290677) roll requested 2024-12-09T08:18:10,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290683 2024-12-09T08:18:10,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290677 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290683 2024-12-09T08:18:10,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,689 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=449, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,689 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,689 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290683) roll requested 2024-12-09T08:18:10,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290689 2024-12-09T08:18:10,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290683 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290689 2024-12-09T08:18:10,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,696 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,696 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=450, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,696 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290689) roll requested 2024-12-09T08:18:10,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290696 2024-12-09T08:18:10,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290689 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290696 2024-12-09T08:18:10,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,704 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,704 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=451, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,704 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290696) roll requested 2024-12-09T08:18:10,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290704 2024-12-09T08:18:10,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290696 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290704 2024-12-09T08:18:10,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,711 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,712 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=452, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,712 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290704) roll requested 2024-12-09T08:18:10,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290712 2024-12-09T08:18:10,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290704 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290712 2024-12-09T08:18:10,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,719 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,719 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=453, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,719 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290712) roll requested 2024-12-09T08:18:10,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290719 2024-12-09T08:18:10,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290712 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290719 2024-12-09T08:18:10,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,726 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,726 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=454, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,726 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290719) roll requested 2024-12-09T08:18:10,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290726 2024-12-09T08:18:10,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290719 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290726 2024-12-09T08:18:10,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,734 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,734 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=455, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,734 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290726) roll requested 2024-12-09T08:18:10,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290734 2024-12-09T08:18:10,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290726 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290734 2024-12-09T08:18:10,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,742 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,742 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=456, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,743 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290734) roll requested 2024-12-09T08:18:10,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290743 2024-12-09T08:18:10,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290734 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290743 2024-12-09T08:18:10,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,751 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,751 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=457, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,751 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290743) roll requested 2024-12-09T08:18:10,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290751 2024-12-09T08:18:10,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290743 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290751 2024-12-09T08:18:10,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,760 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,760 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=458, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,760 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290751) roll requested 2024-12-09T08:18:10,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290760 2024-12-09T08:18:10,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290751 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290760 2024-12-09T08:18:10,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,768 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,768 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=459, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,768 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290760) roll requested 2024-12-09T08:18:10,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290768 2024-12-09T08:18:10,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290760 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290768 2024-12-09T08:18:10,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,775 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,776 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=460, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,776 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290768) roll requested 2024-12-09T08:18:10,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290776 2024-12-09T08:18:10,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290768 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290776 2024-12-09T08:18:10,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,784 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,784 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=461, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,784 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290776) roll requested 2024-12-09T08:18:10,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290785 2024-12-09T08:18:10,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290776 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290785 2024-12-09T08:18:10,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,793 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,793 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=462, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,794 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290785) roll requested 2024-12-09T08:18:10,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290794 2024-12-09T08:18:10,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290785 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290794 2024-12-09T08:18:10,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,802 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,802 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=463, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,802 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290794) roll requested 2024-12-09T08:18:10,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290802 2024-12-09T08:18:10,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290794 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290802 2024-12-09T08:18:10,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,811 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,811 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=464, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,812 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290802) roll requested 2024-12-09T08:18:10,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290812 2024-12-09T08:18:10,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290802 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290812 2024-12-09T08:18:10,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,821 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=465, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,821 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,821 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290812) roll requested 2024-12-09T08:18:10,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290821 2024-12-09T08:18:10,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290812 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290821 2024-12-09T08:18:10,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,829 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,829 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=466, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,829 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290821) roll requested 2024-12-09T08:18:10,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290830 2024-12-09T08:18:10,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290821 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290830 2024-12-09T08:18:10,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,837 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,838 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=467, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,838 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290830) roll requested 2024-12-09T08:18:10,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290838 2024-12-09T08:18:10,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290830 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290838 2024-12-09T08:18:10,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,847 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,847 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=468, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,847 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290838) roll requested 2024-12-09T08:18:10,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290847 2024-12-09T08:18:10,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290838 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290847 2024-12-09T08:18:10,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,855 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,855 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=469, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,855 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290847) roll requested 2024-12-09T08:18:10,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290855 2024-12-09T08:18:10,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290847 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290855 2024-12-09T08:18:10,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,864 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,864 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=470, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,864 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290855) roll requested 2024-12-09T08:18:10,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290865 2024-12-09T08:18:10,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290855 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290865 2024-12-09T08:18:10,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,872 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,872 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=471, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,872 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290865) roll requested 2024-12-09T08:18:10,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290872 2024-12-09T08:18:10,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290865 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290872 2024-12-09T08:18:10,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,879 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,879 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=472, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,879 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290872) roll requested 2024-12-09T08:18:10,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290879 2024-12-09T08:18:10,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290872 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290879 2024-12-09T08:18:10,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,887 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,887 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=473, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,887 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290879) roll requested 2024-12-09T08:18:10,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290887 2024-12-09T08:18:10,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290879 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290887 2024-12-09T08:18:10,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,895 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,895 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=474, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,896 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290887) roll requested 2024-12-09T08:18:10,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290896 2024-12-09T08:18:10,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290887 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290896 2024-12-09T08:18:10,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,905 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,905 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=475, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,905 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290896) roll requested 2024-12-09T08:18:10,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290905 2024-12-09T08:18:10,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290896 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290905 2024-12-09T08:18:10,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,914 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,915 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=476, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,915 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290905) roll requested 2024-12-09T08:18:10,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290915 2024-12-09T08:18:10,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290905 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290915 2024-12-09T08:18:10,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,922 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,922 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=477, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,922 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290915) roll requested 2024-12-09T08:18:10,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290922 2024-12-09T08:18:10,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290915 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290922 2024-12-09T08:18:10,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,930 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,930 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=478, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,930 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290922) roll requested 2024-12-09T08:18:10,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290930 2024-12-09T08:18:10,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290922 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290930 2024-12-09T08:18:10,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,937 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,937 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=479, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,938 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290930) roll requested 2024-12-09T08:18:10,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290938 2024-12-09T08:18:10,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290930 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290938 2024-12-09T08:18:10,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,943 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,943 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=480, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,943 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290938) roll requested 2024-12-09T08:18:10,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290943 2024-12-09T08:18:10,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290938 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290943 2024-12-09T08:18:10,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,948 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,948 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=481, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,948 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290943) roll requested 2024-12-09T08:18:10,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290949 2024-12-09T08:18:10,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290943 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290949 2024-12-09T08:18:10,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,955 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,955 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=482, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,955 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290949) roll requested 2024-12-09T08:18:10,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290955 2024-12-09T08:18:10,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290949 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290955 2024-12-09T08:18:10,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,962 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,962 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=483, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,962 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290955) roll requested 2024-12-09T08:18:10,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290962 2024-12-09T08:18:10,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290955 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290962 2024-12-09T08:18:10,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,968 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,968 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=484, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,968 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290962) roll requested 2024-12-09T08:18:10,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290968 2024-12-09T08:18:10,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290962 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290968 2024-12-09T08:18:10,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,973 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,973 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=485, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,973 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290968) roll requested 2024-12-09T08:18:10,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290973 2024-12-09T08:18:10,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290968 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290973 2024-12-09T08:18:10,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,978 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,978 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=486, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,978 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290973) roll requested 2024-12-09T08:18:10,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290978 2024-12-09T08:18:10,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290973 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290978 2024-12-09T08:18:10,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,985 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,985 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=487, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,986 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290978) roll requested 2024-12-09T08:18:10,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290986 2024-12-09T08:18:10,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:10,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290978 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290986 2024-12-09T08:18:10,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:10,998 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,998 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=488, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:10,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:10,998 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:10,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290986) roll requested 2024-12-09T08:18:10,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732290998 2024-12-09T08:18:11,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290986 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290998 2024-12-09T08:18:11,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,003 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,003 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=489, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,003 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732290998) roll requested 2024-12-09T08:18:11,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291003 2024-12-09T08:18:11,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732290998 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291003 2024-12-09T08:18:11,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,008 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,008 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=490, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,008 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291003) roll requested 2024-12-09T08:18:11,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291008 2024-12-09T08:18:11,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291003 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291008 2024-12-09T08:18:11,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,015 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,015 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=491, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,015 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291008) roll requested 2024-12-09T08:18:11,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291015 2024-12-09T08:18:11,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291008 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291015 2024-12-09T08:18:11,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,022 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,022 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=492, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,022 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291015) roll requested 2024-12-09T08:18:11,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291022 2024-12-09T08:18:11,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291015 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291022 2024-12-09T08:18:11,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,031 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,031 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=493, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,031 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291022) roll requested 2024-12-09T08:18:11,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291031 2024-12-09T08:18:11,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291022 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291031 2024-12-09T08:18:11,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,038 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,038 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=494, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,038 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291031) roll requested 2024-12-09T08:18:11,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291038 2024-12-09T08:18:11,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291031 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291038 2024-12-09T08:18:11,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,049 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,049 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=495, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,049 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291038) roll requested 2024-12-09T08:18:11,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291050 2024-12-09T08:18:11,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291038 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291050 2024-12-09T08:18:11,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,056 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,056 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=496, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,056 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291050) roll requested 2024-12-09T08:18:11,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291056 2024-12-09T08:18:11,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291050 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291056 2024-12-09T08:18:11,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,061 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,061 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=497, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,061 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291056) roll requested 2024-12-09T08:18:11,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291061 2024-12-09T08:18:11,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291056 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291061 2024-12-09T08:18:11,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,066 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,066 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=498, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,066 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291061) roll requested 2024-12-09T08:18:11,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291066 2024-12-09T08:18:11,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291061 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291066 2024-12-09T08:18:11,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,071 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,071 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=499, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,071 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291066) roll requested 2024-12-09T08:18:11,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291071 2024-12-09T08:18:11,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291066 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291071 2024-12-09T08:18:11,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,075 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,075 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=500, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,075 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291071) roll requested 2024-12-09T08:18:11,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291075 2024-12-09T08:18:11,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291071 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291075 2024-12-09T08:18:11,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,082 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,082 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=501, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,082 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291075) roll requested 2024-12-09T08:18:11,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291082 2024-12-09T08:18:11,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291075 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291082 2024-12-09T08:18:11,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,088 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,088 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=502, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,089 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291082) roll requested 2024-12-09T08:18:11,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291089 2024-12-09T08:18:11,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291082 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291089 2024-12-09T08:18:11,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,098 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,098 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=503, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,098 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291089) roll requested 2024-12-09T08:18:11,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291098 2024-12-09T08:18:11,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291089 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291098 2024-12-09T08:18:11,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,107 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,107 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=504, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,107 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291098) roll requested 2024-12-09T08:18:11,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291107 2024-12-09T08:18:11,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291098 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291107 2024-12-09T08:18:11,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,115 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,115 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=505, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,116 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291107) roll requested 2024-12-09T08:18:11,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291116 2024-12-09T08:18:11,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291107 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291116 2024-12-09T08:18:11,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,124 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,125 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=506, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,125 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291116) roll requested 2024-12-09T08:18:11,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291125 2024-12-09T08:18:11,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291116 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291125 2024-12-09T08:18:11,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,133 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,133 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=507, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,133 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291125) roll requested 2024-12-09T08:18:11,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291133 2024-12-09T08:18:11,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291125 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291133 2024-12-09T08:18:11,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,139 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,139 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=508, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,139 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291133) roll requested 2024-12-09T08:18:11,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291139 2024-12-09T08:18:11,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291133 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291139 2024-12-09T08:18:11,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,146 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,146 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=509, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,146 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291139) roll requested 2024-12-09T08:18:11,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291146 2024-12-09T08:18:11,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291139 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291146 2024-12-09T08:18:11,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,154 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,154 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=510, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,154 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291146) roll requested 2024-12-09T08:18:11,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291154 2024-12-09T08:18:11,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291146 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291154 2024-12-09T08:18:11,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,162 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=511, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,162 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,162 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291154) roll requested 2024-12-09T08:18:11,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291162 2024-12-09T08:18:11,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291154 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291162 2024-12-09T08:18:11,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,171 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=512, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,171 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,171 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291162) roll requested 2024-12-09T08:18:11,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291172 2024-12-09T08:18:11,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291162 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291172 2024-12-09T08:18:11,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,179 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,179 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=513, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,179 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291172) roll requested 2024-12-09T08:18:11,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291179 2024-12-09T08:18:11,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291172 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291179 2024-12-09T08:18:11,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,187 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,187 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=514, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,187 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291179) roll requested 2024-12-09T08:18:11,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291187 2024-12-09T08:18:11,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291179 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291187 2024-12-09T08:18:11,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,196 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,196 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=515, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,197 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291187) roll requested 2024-12-09T08:18:11,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291197 2024-12-09T08:18:11,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291187 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291197 2024-12-09T08:18:11,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,205 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,205 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=516, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,205 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291197) roll requested 2024-12-09T08:18:11,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291205 2024-12-09T08:18:11,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291197 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291205 2024-12-09T08:18:11,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,211 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,211 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=517, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,211 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291205) roll requested 2024-12-09T08:18:11,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291211 2024-12-09T08:18:11,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291205 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291211 2024-12-09T08:18:11,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,217 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,218 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=518, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,218 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291211) roll requested 2024-12-09T08:18:11,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291218 2024-12-09T08:18:11,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291211 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291218 2024-12-09T08:18:11,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,225 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,225 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=519, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,225 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291218) roll requested 2024-12-09T08:18:11,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291226 2024-12-09T08:18:11,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291218 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291226 2024-12-09T08:18:11,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,233 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,233 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=520, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,233 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291226) roll requested 2024-12-09T08:18:11,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291233 2024-12-09T08:18:11,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291226 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291233 2024-12-09T08:18:11,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,240 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,240 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=521, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,241 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291233) roll requested 2024-12-09T08:18:11,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291241 2024-12-09T08:18:11,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291233 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291241 2024-12-09T08:18:11,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,248 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,248 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=522, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,248 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291241) roll requested 2024-12-09T08:18:11,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291248 2024-12-09T08:18:11,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291241 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291248 2024-12-09T08:18:11,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,256 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,256 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=523, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,256 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291248) roll requested 2024-12-09T08:18:11,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291256 2024-12-09T08:18:11,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291248 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291256 2024-12-09T08:18:11,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,265 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,266 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=524, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,266 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291256) roll requested 2024-12-09T08:18:11,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291266 2024-12-09T08:18:11,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291256 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291266 2024-12-09T08:18:11,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,272 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,273 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=525, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,273 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291266) roll requested 2024-12-09T08:18:11,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291273 2024-12-09T08:18:11,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291266 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291273 2024-12-09T08:18:11,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,280 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,280 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=526, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,280 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291273) roll requested 2024-12-09T08:18:11,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291281 2024-12-09T08:18:11,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291273 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291281 2024-12-09T08:18:11,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,288 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,288 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=527, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,288 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291281) roll requested 2024-12-09T08:18:11,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291288 2024-12-09T08:18:11,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291281 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291288 2024-12-09T08:18:11,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,296 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,296 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=528, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,296 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291288) roll requested 2024-12-09T08:18:11,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291296 2024-12-09T08:18:11,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291288 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291296 2024-12-09T08:18:11,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,305 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,305 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=529, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,305 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291296) roll requested 2024-12-09T08:18:11,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291305 2024-12-09T08:18:11,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291296 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291305 2024-12-09T08:18:11,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,311 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,311 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=530, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,311 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291305) roll requested 2024-12-09T08:18:11,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291312 2024-12-09T08:18:11,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291305 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291312 2024-12-09T08:18:11,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,319 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,319 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=531, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,319 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291312) roll requested 2024-12-09T08:18:11,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291319 2024-12-09T08:18:11,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291312 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291319 2024-12-09T08:18:11,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,328 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,328 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=532, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,328 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291319) roll requested 2024-12-09T08:18:11,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291329 2024-12-09T08:18:11,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291319 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291329 2024-12-09T08:18:11,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,337 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,337 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=533, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,337 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291329) roll requested 2024-12-09T08:18:11,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291337 2024-12-09T08:18:11,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291329 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291337 2024-12-09T08:18:11,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,344 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,344 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=534, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,344 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291337) roll requested 2024-12-09T08:18:11,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291344 2024-12-09T08:18:11,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291337 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291344 2024-12-09T08:18:11,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,350 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,351 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=535, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,351 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291344) roll requested 2024-12-09T08:18:11,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291351 2024-12-09T08:18:11,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291344 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291351 2024-12-09T08:18:11,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,359 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,359 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=536, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,359 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291351) roll requested 2024-12-09T08:18:11,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291359 2024-12-09T08:18:11,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291351 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291359 2024-12-09T08:18:11,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,369 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,369 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=537, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,369 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291359) roll requested 2024-12-09T08:18:11,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291369 2024-12-09T08:18:11,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291359 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291369 2024-12-09T08:18:11,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,378 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,378 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=538, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,378 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291369) roll requested 2024-12-09T08:18:11,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291378 2024-12-09T08:18:11,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291369 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291378 2024-12-09T08:18:11,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,386 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,386 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=539, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,386 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291378) roll requested 2024-12-09T08:18:11,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291386 2024-12-09T08:18:11,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291378 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291386 2024-12-09T08:18:11,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,397 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=540, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,397 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,397 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291386) roll requested 2024-12-09T08:18:11,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291398 2024-12-09T08:18:11,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291386 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291398 2024-12-09T08:18:11,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,407 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,407 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=541, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,407 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291398) roll requested 2024-12-09T08:18:11,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291407 2024-12-09T08:18:11,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291398 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291407 2024-12-09T08:18:11,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,416 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,417 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=542, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,417 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291407) roll requested 2024-12-09T08:18:11,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291417 2024-12-09T08:18:11,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,424 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,424 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291407 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291417 2024-12-09T08:18:11,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,424 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,424 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=543, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,425 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291417) roll requested 2024-12-09T08:18:11,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291425 2024-12-09T08:18:11,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291417 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291425 2024-12-09T08:18:11,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,429 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,429 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=544, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,430 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291425) roll requested 2024-12-09T08:18:11,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291430 2024-12-09T08:18:11,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291425 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291430 2024-12-09T08:18:11,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,435 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,435 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=545, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,435 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291430) roll requested 2024-12-09T08:18:11,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291435 2024-12-09T08:18:11,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291430 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291435 2024-12-09T08:18:11,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,442 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,442 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=546, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,442 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291435) roll requested 2024-12-09T08:18:11,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291442 2024-12-09T08:18:11,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291435 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291442 2024-12-09T08:18:11,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,449 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,449 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=547, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,449 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291442) roll requested 2024-12-09T08:18:11,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291450 2024-12-09T08:18:11,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291442 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291450 2024-12-09T08:18:11,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,457 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,457 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=548, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,457 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291450) roll requested 2024-12-09T08:18:11,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291457 2024-12-09T08:18:11,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291450 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291457 2024-12-09T08:18:11,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,464 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,465 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=549, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,465 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291457) roll requested 2024-12-09T08:18:11,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291465 2024-12-09T08:18:11,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291457 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291465 2024-12-09T08:18:11,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,474 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,474 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=550, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,474 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291465) roll requested 2024-12-09T08:18:11,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291474 2024-12-09T08:18:11,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291465 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291474 2024-12-09T08:18:11,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,483 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,483 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=551, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,483 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291474) roll requested 2024-12-09T08:18:11,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291483 2024-12-09T08:18:11,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291474 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291483 2024-12-09T08:18:11,493 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,493 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=552, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,493 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291483) roll requested 2024-12-09T08:18:11,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291494 2024-12-09T08:18:11,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291483 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291494 2024-12-09T08:18:11,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,503 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,503 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=553, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,503 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291494) roll requested 2024-12-09T08:18:11,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291503 2024-12-09T08:18:11,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291494 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291503 2024-12-09T08:18:11,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,512 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,512 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=554, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,513 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291503) roll requested 2024-12-09T08:18:11,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291513 2024-12-09T08:18:11,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291503 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291513 2024-12-09T08:18:11,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,521 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,521 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=555, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,522 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291513) roll requested 2024-12-09T08:18:11,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291522 2024-12-09T08:18:11,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291513 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291522 2024-12-09T08:18:11,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,527 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,527 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=556, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,528 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291522) roll requested 2024-12-09T08:18:11,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291528 2024-12-09T08:18:11,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291522 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291528 2024-12-09T08:18:11,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,534 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,534 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=557, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,534 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291528) roll requested 2024-12-09T08:18:11,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291534 2024-12-09T08:18:11,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291528 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291534 2024-12-09T08:18:11,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,542 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,542 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=558, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,542 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291534) roll requested 2024-12-09T08:18:11,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291542 2024-12-09T08:18:11,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291534 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291542 2024-12-09T08:18:11,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,550 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,550 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=559, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,550 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291542) roll requested 2024-12-09T08:18:11,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291550 2024-12-09T08:18:11,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291542 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291550 2024-12-09T08:18:11,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,558 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,559 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=560, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,559 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291550) roll requested 2024-12-09T08:18:11,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291559 2024-12-09T08:18:11,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291550 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291559 2024-12-09T08:18:11,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,567 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,567 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=561, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,567 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291559) roll requested 2024-12-09T08:18:11,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291567 2024-12-09T08:18:11,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291559 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291567 2024-12-09T08:18:11,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,577 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,577 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=562, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,578 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291567) roll requested 2024-12-09T08:18:11,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291578 2024-12-09T08:18:11,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291567 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291578 2024-12-09T08:18:11,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,583 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,583 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=563, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,584 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291578) roll requested 2024-12-09T08:18:11,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291584 2024-12-09T08:18:11,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291578 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291584 2024-12-09T08:18:11,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,589 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,589 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=564, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,589 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291584) roll requested 2024-12-09T08:18:11,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291589 2024-12-09T08:18:11,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291584 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291589 2024-12-09T08:18:11,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,595 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,595 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=565, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,595 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291589) roll requested 2024-12-09T08:18:11,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291596 2024-12-09T08:18:11,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291589 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291596 2024-12-09T08:18:11,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,602 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,602 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=566, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,602 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291596) roll requested 2024-12-09T08:18:11,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291602 2024-12-09T08:18:11,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291596 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291602 2024-12-09T08:18:11,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,609 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,609 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=567, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,610 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291602) roll requested 2024-12-09T08:18:11,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291610 2024-12-09T08:18:11,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291602 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291610 2024-12-09T08:18:11,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,618 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,618 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=568, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,618 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291610) roll requested 2024-12-09T08:18:11,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291618 2024-12-09T08:18:11,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291610 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291618 2024-12-09T08:18:11,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,625 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,625 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=569, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,625 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291618) roll requested 2024-12-09T08:18:11,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291626 2024-12-09T08:18:11,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291618 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291626 2024-12-09T08:18:11,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,633 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,634 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=570, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,634 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291626) roll requested 2024-12-09T08:18:11,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291634 2024-12-09T08:18:11,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291626 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291634 2024-12-09T08:18:11,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,643 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,643 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=571, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,643 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291634) roll requested 2024-12-09T08:18:11,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291643 2024-12-09T08:18:11,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291634 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291643 2024-12-09T08:18:11,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,651 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,652 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=572, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,652 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291643) roll requested 2024-12-09T08:18:11,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291652 2024-12-09T08:18:11,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291643 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291652 2024-12-09T08:18:11,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,660 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,660 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=573, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,660 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291652) roll requested 2024-12-09T08:18:11,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291660 2024-12-09T08:18:11,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291652 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291660 2024-12-09T08:18:11,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,669 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,669 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=574, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,669 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291660) roll requested 2024-12-09T08:18:11,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291670 2024-12-09T08:18:11,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291660 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291670 2024-12-09T08:18:11,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,678 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,678 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=575, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,678 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291670) roll requested 2024-12-09T08:18:11,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291678 2024-12-09T08:18:11,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291670 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291678 2024-12-09T08:18:11,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,684 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,684 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=576, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,685 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291678) roll requested 2024-12-09T08:18:11,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291685 2024-12-09T08:18:11,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291678 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291685 2024-12-09T08:18:11,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,695 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,695 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=577, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,695 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291685) roll requested 2024-12-09T08:18:11,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291695 2024-12-09T08:18:11,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291685 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291695 2024-12-09T08:18:11,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,702 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,703 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=578, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,703 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291695) roll requested 2024-12-09T08:18:11,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291703 2024-12-09T08:18:11,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291695 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291703 2024-12-09T08:18:11,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,709 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,709 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=579, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,709 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291703) roll requested 2024-12-09T08:18:11,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291709 2024-12-09T08:18:11,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291703 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291709 2024-12-09T08:18:11,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,715 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,715 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=580, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,715 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291709) roll requested 2024-12-09T08:18:11,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291715 2024-12-09T08:18:11,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291715 2024-12-09T08:18:11,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,724 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,724 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=581, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,724 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291715) roll requested 2024-12-09T08:18:11,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291724 2024-12-09T08:18:11,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291715 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291724 2024-12-09T08:18:11,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,729 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,730 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=582, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,730 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291724) roll requested 2024-12-09T08:18:11,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291730 2024-12-09T08:18:11,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291724 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291730 2024-12-09T08:18:11,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,739 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,739 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=583, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,739 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291730) roll requested 2024-12-09T08:18:11,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291739 2024-12-09T08:18:11,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291730 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291739 2024-12-09T08:18:11,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,745 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,745 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=584, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,746 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291739) roll requested 2024-12-09T08:18:11,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291746 2024-12-09T08:18:11,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291739 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291746 2024-12-09T08:18:11,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,751 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,751 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=585, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,751 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291746) roll requested 2024-12-09T08:18:11,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291751 2024-12-09T08:18:11,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291746 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291751 2024-12-09T08:18:11,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,756 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,757 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=586, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,757 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291751) roll requested 2024-12-09T08:18:11,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291757 2024-12-09T08:18:11,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291751 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291757 2024-12-09T08:18:11,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,764 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,764 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=587, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,764 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291757) roll requested 2024-12-09T08:18:11,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291764 2024-12-09T08:18:11,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291757 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291764 2024-12-09T08:18:11,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,773 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,773 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=588, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,773 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291764) roll requested 2024-12-09T08:18:11,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291773 2024-12-09T08:18:11,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291764 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291773 2024-12-09T08:18:11,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,778 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,778 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=589, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,778 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291773) roll requested 2024-12-09T08:18:11,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291778 2024-12-09T08:18:11,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291773 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291778 2024-12-09T08:18:11,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,785 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,785 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=590, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,785 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291778) roll requested 2024-12-09T08:18:11,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291785 2024-12-09T08:18:11,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291778 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291785 2024-12-09T08:18:11,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,790 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,790 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=591, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,790 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291785) roll requested 2024-12-09T08:18:11,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291790 2024-12-09T08:18:11,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291785 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291790 2024-12-09T08:18:11,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,797 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,797 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=592, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,797 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291790) roll requested 2024-12-09T08:18:11,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291797 2024-12-09T08:18:11,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291790 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291797 2024-12-09T08:18:11,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,802 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,802 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=593, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,802 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291797) roll requested 2024-12-09T08:18:11,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291802 2024-12-09T08:18:11,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291797 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291802 2024-12-09T08:18:11,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,807 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,807 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=594, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,807 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291802) roll requested 2024-12-09T08:18:11,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291807 2024-12-09T08:18:11,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291802 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291807 2024-12-09T08:18:11,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,813 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,813 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=595, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,813 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291807) roll requested 2024-12-09T08:18:11,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291813 2024-12-09T08:18:11,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291807 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291813 2024-12-09T08:18:11,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,819 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,820 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=596, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,820 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291813) roll requested 2024-12-09T08:18:11,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291820 2024-12-09T08:18:11,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291813 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291820 2024-12-09T08:18:11,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,829 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,829 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=597, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,829 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291820) roll requested 2024-12-09T08:18:11,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291829 2024-12-09T08:18:11,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291820 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291829 2024-12-09T08:18:11,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,838 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,838 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=598, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,838 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291829) roll requested 2024-12-09T08:18:11,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291838 2024-12-09T08:18:11,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291829 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291838 2024-12-09T08:18:11,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,846 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,846 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=599, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,846 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291838) roll requested 2024-12-09T08:18:11,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291846 2024-12-09T08:18:11,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291838 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291846 2024-12-09T08:18:11,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,853 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,854 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=600, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,854 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291846) roll requested 2024-12-09T08:18:11,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291854 2024-12-09T08:18:11,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291846 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291854 2024-12-09T08:18:11,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,861 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,861 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=601, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,861 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291854) roll requested 2024-12-09T08:18:11,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291861 2024-12-09T08:18:11,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291854 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291861 2024-12-09T08:18:11,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,869 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,870 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=602, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,870 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291861) roll requested 2024-12-09T08:18:11,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291870 2024-12-09T08:18:11,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291861 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291870 2024-12-09T08:18:11,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,878 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,878 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=603, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,879 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291870) roll requested 2024-12-09T08:18:11,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291879 2024-12-09T08:18:11,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291870 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291879 2024-12-09T08:18:11,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,887 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,887 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=604, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,887 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291879) roll requested 2024-12-09T08:18:11,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291887 2024-12-09T08:18:11,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291879 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291887 2024-12-09T08:18:11,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,895 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,895 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=605, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,895 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291887) roll requested 2024-12-09T08:18:11,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291896 2024-12-09T08:18:11,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291887 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291896 2024-12-09T08:18:11,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,903 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,904 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=606, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,904 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291896) roll requested 2024-12-09T08:18:11,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291904 2024-12-09T08:18:11,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291896 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291904 2024-12-09T08:18:11,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,911 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,911 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=607, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,911 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291904) roll requested 2024-12-09T08:18:11,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291911 2024-12-09T08:18:11,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291904 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291911 2024-12-09T08:18:11,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,920 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,921 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=608, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,921 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291911) roll requested 2024-12-09T08:18:11,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291921 2024-12-09T08:18:11,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291911 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291921 2024-12-09T08:18:11,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,928 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,928 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=609, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,928 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291921) roll requested 2024-12-09T08:18:11,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291928 2024-12-09T08:18:11,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291921 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291928 2024-12-09T08:18:11,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,936 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,936 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=610, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,936 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291928) roll requested 2024-12-09T08:18:11,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291936 2024-12-09T08:18:11,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291928 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291936 2024-12-09T08:18:11,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,943 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,944 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=611, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,944 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291936) roll requested 2024-12-09T08:18:11,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291944 2024-12-09T08:18:11,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291936 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291944 2024-12-09T08:18:11,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,950 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,950 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=612, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,950 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291944) roll requested 2024-12-09T08:18:11,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291950 2024-12-09T08:18:11,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291944 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291950 2024-12-09T08:18:11,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,958 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,958 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=613, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,958 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291950) roll requested 2024-12-09T08:18:11,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291958 2024-12-09T08:18:11,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291950 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291958 2024-12-09T08:18:11,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,967 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,967 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=614, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,968 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291958) roll requested 2024-12-09T08:18:11,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291968 2024-12-09T08:18:11,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291958 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291968 2024-12-09T08:18:11,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,976 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,977 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=615, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,977 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291968) roll requested 2024-12-09T08:18:11,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291977 2024-12-09T08:18:11,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291968 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291977 2024-12-09T08:18:11,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,983 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,983 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=616, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,984 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291977) roll requested 2024-12-09T08:18:11,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291984 2024-12-09T08:18:11,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291977 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291984 2024-12-09T08:18:11,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,991 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,991 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=617, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,991 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291984) roll requested 2024-12-09T08:18:11,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291991 2024-12-09T08:18:11,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:11,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291984 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291991 2024-12-09T08:18:11,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:11,999 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:11,999 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=618, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:11,999 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:11,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291991) roll requested 2024-12-09T08:18:11,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732291999 2024-12-09T08:18:12,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291991 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291999 2024-12-09T08:18:12,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,006 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,006 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=619, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,006 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732291999) roll requested 2024-12-09T08:18:12,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292006 2024-12-09T08:18:12,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732291999 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292006 2024-12-09T08:18:12,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,013 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,014 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=620, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,014 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292006) roll requested 2024-12-09T08:18:12,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292014 2024-12-09T08:18:12,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292006 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292014 2024-12-09T08:18:12,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,021 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,022 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=621, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,022 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292014) roll requested 2024-12-09T08:18:12,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292022 2024-12-09T08:18:12,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292014 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292022 2024-12-09T08:18:12,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,030 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,030 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=622, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,030 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292022) roll requested 2024-12-09T08:18:12,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292030 2024-12-09T08:18:12,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292022 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292030 2024-12-09T08:18:12,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,037 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,038 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=623, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,038 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292030) roll requested 2024-12-09T08:18:12,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292038 2024-12-09T08:18:12,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292030 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292038 2024-12-09T08:18:12,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,045 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,045 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=624, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,045 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292038) roll requested 2024-12-09T08:18:12,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292045 2024-12-09T08:18:12,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292038 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292045 2024-12-09T08:18:12,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,052 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,052 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=625, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,052 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292045) roll requested 2024-12-09T08:18:12,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292052 2024-12-09T08:18:12,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292045 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292052 2024-12-09T08:18:12,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,059 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,060 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=626, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,060 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292052) roll requested 2024-12-09T08:18:12,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292060 2024-12-09T08:18:12,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292052 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292060 2024-12-09T08:18:12,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,069 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,069 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=627, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,069 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292060) roll requested 2024-12-09T08:18:12,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292069 2024-12-09T08:18:12,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292060 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292069 2024-12-09T08:18:12,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,076 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,077 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=628, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,077 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292069) roll requested 2024-12-09T08:18:12,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292077 2024-12-09T08:18:12,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292069 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292077 2024-12-09T08:18:12,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,084 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,084 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=629, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,084 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292077) roll requested 2024-12-09T08:18:12,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292084 2024-12-09T08:18:12,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292077 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292084 2024-12-09T08:18:12,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,093 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,093 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=630, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,093 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292084) roll requested 2024-12-09T08:18:12,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292093 2024-12-09T08:18:12,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292084 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292093 2024-12-09T08:18:12,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,101 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,102 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=631, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,102 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292093) roll requested 2024-12-09T08:18:12,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292102 2024-12-09T08:18:12,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292093 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292102 2024-12-09T08:18:12,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,108 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,108 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=632, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,109 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292102) roll requested 2024-12-09T08:18:12,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292109 2024-12-09T08:18:12,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292102 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292109 2024-12-09T08:18:12,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,113 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,113 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=633, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,113 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292109) roll requested 2024-12-09T08:18:12,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292113 2024-12-09T08:18:12,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292109 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292113 2024-12-09T08:18:12,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,118 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,118 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=634, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,119 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292113) roll requested 2024-12-09T08:18:12,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292119 2024-12-09T08:18:12,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292113 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292119 2024-12-09T08:18:12,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,125 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,125 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=635, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,125 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292119) roll requested 2024-12-09T08:18:12,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292125 2024-12-09T08:18:12,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292119 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292125 2024-12-09T08:18:12,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,132 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,132 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=636, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,132 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292125) roll requested 2024-12-09T08:18:12,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292133 2024-12-09T08:18:12,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292125 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292133 2024-12-09T08:18:12,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,140 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,140 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=637, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,140 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292133) roll requested 2024-12-09T08:18:12,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292140 2024-12-09T08:18:12,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292133 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292140 2024-12-09T08:18:12,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,148 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,149 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=638, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,149 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292140) roll requested 2024-12-09T08:18:12,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292149 2024-12-09T08:18:12,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292140 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292149 2024-12-09T08:18:12,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,156 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,156 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=639, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,156 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292149) roll requested 2024-12-09T08:18:12,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292156 2024-12-09T08:18:12,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292149 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292156 2024-12-09T08:18:12,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,163 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,163 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=640, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,163 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292156) roll requested 2024-12-09T08:18:12,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292163 2024-12-09T08:18:12,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292156 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292163 2024-12-09T08:18:12,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,173 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,173 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=641, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,173 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292163) roll requested 2024-12-09T08:18:12,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292173 2024-12-09T08:18:12,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292163 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292173 2024-12-09T08:18:12,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,180 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,180 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=642, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,181 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292173) roll requested 2024-12-09T08:18:12,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292181 2024-12-09T08:18:12,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292173 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292181 2024-12-09T08:18:12,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,185 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,185 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=643, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,186 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292181) roll requested 2024-12-09T08:18:12,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292186 2024-12-09T08:18:12,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292181 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292186 2024-12-09T08:18:12,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,195 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,195 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=644, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,195 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292186) roll requested 2024-12-09T08:18:12,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292195 2024-12-09T08:18:12,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292186 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292195 2024-12-09T08:18:12,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,200 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,200 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=645, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,200 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292195) roll requested 2024-12-09T08:18:12,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292200 2024-12-09T08:18:12,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292195 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292200 2024-12-09T08:18:12,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,205 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,205 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=646, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,205 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292200) roll requested 2024-12-09T08:18:12,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292205 2024-12-09T08:18:12,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292200 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292205 2024-12-09T08:18:12,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,213 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,213 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=647, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,213 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292205) roll requested 2024-12-09T08:18:12,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292213 2024-12-09T08:18:12,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292205 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292213 2024-12-09T08:18:12,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,220 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,220 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=648, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,220 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292213) roll requested 2024-12-09T08:18:12,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292221 2024-12-09T08:18:12,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292213 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292221 2024-12-09T08:18:12,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,230 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,231 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=649, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,231 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292221) roll requested 2024-12-09T08:18:12,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292231 2024-12-09T08:18:12,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292221 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292231 2024-12-09T08:18:12,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,239 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,239 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=650, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,240 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292231) roll requested 2024-12-09T08:18:12,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292240 2024-12-09T08:18:12,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292231 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292240 2024-12-09T08:18:12,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,247 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,247 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=651, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,247 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292240) roll requested 2024-12-09T08:18:12,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292247 2024-12-09T08:18:12,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292240 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292247 2024-12-09T08:18:12,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,255 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,256 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=652, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,256 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292247) roll requested 2024-12-09T08:18:12,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292256 2024-12-09T08:18:12,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292247 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292256 2024-12-09T08:18:12,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,261 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,261 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=653, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,261 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292256) roll requested 2024-12-09T08:18:12,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292261 2024-12-09T08:18:12,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292256 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292261 2024-12-09T08:18:12,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,266 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,266 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=654, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,266 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292261) roll requested 2024-12-09T08:18:12,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292266 2024-12-09T08:18:12,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292261 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292266 2024-12-09T08:18:12,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,271 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,271 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=655, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,271 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292266) roll requested 2024-12-09T08:18:12,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292272 2024-12-09T08:18:12,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292266 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292272 2024-12-09T08:18:12,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,277 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,277 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=656, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,277 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292272) roll requested 2024-12-09T08:18:12,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292277 2024-12-09T08:18:12,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292272 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292277 2024-12-09T08:18:12,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,282 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,282 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=657, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,282 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292277) roll requested 2024-12-09T08:18:12,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292282 2024-12-09T08:18:12,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292277 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292282 2024-12-09T08:18:12,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,287 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,287 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=658, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,288 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292282) roll requested 2024-12-09T08:18:12,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292288 2024-12-09T08:18:12,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292282 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292288 2024-12-09T08:18:12,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,294 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,294 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=659, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,294 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292288) roll requested 2024-12-09T08:18:12,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292294 2024-12-09T08:18:12,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292288 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292294 2024-12-09T08:18:12,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,299 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,299 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=660, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,299 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292294) roll requested 2024-12-09T08:18:12,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292300 2024-12-09T08:18:12,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292294 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292300 2024-12-09T08:18:12,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,306 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,306 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=661, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,306 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292300) roll requested 2024-12-09T08:18:12,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292306 2024-12-09T08:18:12,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292300 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292306 2024-12-09T08:18:12,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,311 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,311 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=662, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,311 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292306) roll requested 2024-12-09T08:18:12,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292311 2024-12-09T08:18:12,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292306 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292311 2024-12-09T08:18:12,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,317 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,317 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=663, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,317 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292311) roll requested 2024-12-09T08:18:12,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292317 2024-12-09T08:18:12,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292311 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292317 2024-12-09T08:18:12,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,326 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,326 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=664, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,326 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292317) roll requested 2024-12-09T08:18:12,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292326 2024-12-09T08:18:12,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292317 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292326 2024-12-09T08:18:12,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,331 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,331 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=665, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,331 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292326) roll requested 2024-12-09T08:18:12,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292331 2024-12-09T08:18:12,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292326 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292331 2024-12-09T08:18:12,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,336 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,336 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=666, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,336 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292331) roll requested 2024-12-09T08:18:12,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292336 2024-12-09T08:18:12,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292331 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292336 2024-12-09T08:18:12,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,342 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,342 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=667, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,342 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292336) roll requested 2024-12-09T08:18:12,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292342 2024-12-09T08:18:12,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292336 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292342 2024-12-09T08:18:12,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,348 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,348 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=668, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,348 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292342) roll requested 2024-12-09T08:18:12,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292348 2024-12-09T08:18:12,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292342 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292348 2024-12-09T08:18:12,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,353 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,353 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=669, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,354 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292348) roll requested 2024-12-09T08:18:12,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292354 2024-12-09T08:18:12,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292348 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292354 2024-12-09T08:18:12,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,358 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,359 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=670, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,359 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292354) roll requested 2024-12-09T08:18:12,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292359 2024-12-09T08:18:12,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292354 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292359 2024-12-09T08:18:12,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,364 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,364 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=671, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,364 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292359) roll requested 2024-12-09T08:18:12,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292364 2024-12-09T08:18:12,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292359 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292364 2024-12-09T08:18:12,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,369 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,369 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=672, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,369 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292364) roll requested 2024-12-09T08:18:12,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292369 2024-12-09T08:18:12,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292364 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292369 2024-12-09T08:18:12,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,377 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,377 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=673, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,377 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292369) roll requested 2024-12-09T08:18:12,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292377 2024-12-09T08:18:12,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292369 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292377 2024-12-09T08:18:12,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,382 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,382 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=674, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,383 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292377) roll requested 2024-12-09T08:18:12,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292383 2024-12-09T08:18:12,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292377 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292383 2024-12-09T08:18:12,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,388 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,388 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=675, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,388 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292383) roll requested 2024-12-09T08:18:12,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292389 2024-12-09T08:18:12,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292383 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292389 2024-12-09T08:18:12,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,394 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,394 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=676, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,394 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292389) roll requested 2024-12-09T08:18:12,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292394 2024-12-09T08:18:12,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292389 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292394 2024-12-09T08:18:12,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,401 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,401 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=677, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,401 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292394) roll requested 2024-12-09T08:18:12,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292401 2024-12-09T08:18:12,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292394 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292401 2024-12-09T08:18:12,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,412 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,412 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=678, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,412 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292401) roll requested 2024-12-09T08:18:12,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292413 2024-12-09T08:18:12,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292401 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292413 2024-12-09T08:18:12,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,417 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,418 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=679, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,418 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292413) roll requested 2024-12-09T08:18:12,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292418 2024-12-09T08:18:12,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292413 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292418 2024-12-09T08:18:12,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,422 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,423 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=680, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,423 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292418) roll requested 2024-12-09T08:18:12,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292423 2024-12-09T08:18:12,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292418 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292423 2024-12-09T08:18:12,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,427 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,427 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=681, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,427 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292423) roll requested 2024-12-09T08:18:12,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292427 2024-12-09T08:18:12,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292423 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292427 2024-12-09T08:18:12,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,432 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,432 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=682, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,432 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292427) roll requested 2024-12-09T08:18:12,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292433 2024-12-09T08:18:12,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292427 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292433 2024-12-09T08:18:12,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,439 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,439 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=683, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,439 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292433) roll requested 2024-12-09T08:18:12,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292439 2024-12-09T08:18:12,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292433 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292439 2024-12-09T08:18:12,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,446 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,446 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=684, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,446 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292439) roll requested 2024-12-09T08:18:12,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292446 2024-12-09T08:18:12,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292439 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292446 2024-12-09T08:18:12,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,453 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,454 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=685, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,454 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292446) roll requested 2024-12-09T08:18:12,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292454 2024-12-09T08:18:12,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292446 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292454 2024-12-09T08:18:12,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,460 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,460 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=686, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,460 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292454) roll requested 2024-12-09T08:18:12,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292460 2024-12-09T08:18:12,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292454 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292460 2024-12-09T08:18:12,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,467 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,467 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=687, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,467 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292460) roll requested 2024-12-09T08:18:12,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292467 2024-12-09T08:18:12,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292460 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292467 2024-12-09T08:18:12,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,475 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,475 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=688, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,475 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292467) roll requested 2024-12-09T08:18:12,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292475 2024-12-09T08:18:12,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292467 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292475 2024-12-09T08:18:12,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,484 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,484 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=689, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,484 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292475) roll requested 2024-12-09T08:18:12,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292485 2024-12-09T08:18:12,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292475 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292485 2024-12-09T08:18:12,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,493 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,493 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=690, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,494 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292485) roll requested 2024-12-09T08:18:12,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292494 2024-12-09T08:18:12,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292485 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292494 2024-12-09T08:18:12,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,503 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,503 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=691, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,503 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292494) roll requested 2024-12-09T08:18:12,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292504 2024-12-09T08:18:12,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292494 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292504 2024-12-09T08:18:12,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,513 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,513 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=692, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,513 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292504) roll requested 2024-12-09T08:18:12,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292513 2024-12-09T08:18:12,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292504 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292513 2024-12-09T08:18:12,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,518 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,518 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=693, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,518 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292513) roll requested 2024-12-09T08:18:12,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292518 2024-12-09T08:18:12,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292513 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292518 2024-12-09T08:18:12,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,525 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,525 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=694, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,525 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292518) roll requested 2024-12-09T08:18:12,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292525 2024-12-09T08:18:12,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292518 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292525 2024-12-09T08:18:12,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,533 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,533 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=695, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,533 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292525) roll requested 2024-12-09T08:18:12,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292533 2024-12-09T08:18:12,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292525 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292533 2024-12-09T08:18:12,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,541 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,541 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=696, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,541 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292533) roll requested 2024-12-09T08:18:12,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292541 2024-12-09T08:18:12,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292533 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292541 2024-12-09T08:18:12,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,547 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,547 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=697, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,547 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292541) roll requested 2024-12-09T08:18:12,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292547 2024-12-09T08:18:12,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292541 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292547 2024-12-09T08:18:12,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,552 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,552 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=698, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,552 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292547) roll requested 2024-12-09T08:18:12,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292552 2024-12-09T08:18:12,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292547 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292552 2024-12-09T08:18:12,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,558 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,558 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=699, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,558 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292552) roll requested 2024-12-09T08:18:12,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292558 2024-12-09T08:18:12,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292552 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292558 2024-12-09T08:18:12,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,565 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,565 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=700, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,565 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292558) roll requested 2024-12-09T08:18:12,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292565 2024-12-09T08:18:12,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292558 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292565 2024-12-09T08:18:12,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,571 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,571 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=701, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,571 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292565) roll requested 2024-12-09T08:18:12,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292571 2024-12-09T08:18:12,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292565 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292571 2024-12-09T08:18:12,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,576 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,576 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=702, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,576 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292571) roll requested 2024-12-09T08:18:12,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292576 2024-12-09T08:18:12,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292571 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292576 2024-12-09T08:18:12,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,582 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,582 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=703, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,582 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292576) roll requested 2024-12-09T08:18:12,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292582 2024-12-09T08:18:12,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292576 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292582 2024-12-09T08:18:12,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,587 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,587 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=704, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,587 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292582) roll requested 2024-12-09T08:18:12,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292588 2024-12-09T08:18:12,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292582 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292588 2024-12-09T08:18:12,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,593 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,593 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=705, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,594 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292588) roll requested 2024-12-09T08:18:12,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292594 2024-12-09T08:18:12,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292588 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292594 2024-12-09T08:18:12,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,599 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,599 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=706, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,599 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292594) roll requested 2024-12-09T08:18:12,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292599 2024-12-09T08:18:12,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292594 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292599 2024-12-09T08:18:12,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,604 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,604 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=707, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,604 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292599) roll requested 2024-12-09T08:18:12,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292604 2024-12-09T08:18:12,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292599 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292604 2024-12-09T08:18:12,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,612 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,612 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=708, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,612 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292604) roll requested 2024-12-09T08:18:12,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292613 2024-12-09T08:18:12,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292604 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292613 2024-12-09T08:18:12,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,617 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,618 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=709, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,618 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292613) roll requested 2024-12-09T08:18:12,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292618 2024-12-09T08:18:12,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292613 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292618 2024-12-09T08:18:12,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,623 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,623 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=710, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,623 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292618) roll requested 2024-12-09T08:18:12,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292623 2024-12-09T08:18:12,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292618 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292623 2024-12-09T08:18:12,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,628 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,628 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=711, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,628 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292623) roll requested 2024-12-09T08:18:12,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292629 2024-12-09T08:18:12,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292623 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292629 2024-12-09T08:18:12,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,633 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,633 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=712, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,633 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292629) roll requested 2024-12-09T08:18:12,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292633 2024-12-09T08:18:12,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292629 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292633 2024-12-09T08:18:12,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,638 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,638 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=713, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,638 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292633) roll requested 2024-12-09T08:18:12,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292639 2024-12-09T08:18:12,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292633 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292639 2024-12-09T08:18:12,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,647 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,647 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=714, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,647 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292639) roll requested 2024-12-09T08:18:12,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292647 2024-12-09T08:18:12,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292639 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292647 2024-12-09T08:18:12,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,657 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,657 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=715, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,657 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292647) roll requested 2024-12-09T08:18:12,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292657 2024-12-09T08:18:12,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292647 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292657 2024-12-09T08:18:12,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,665 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,665 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=716, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,665 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292657) roll requested 2024-12-09T08:18:12,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292665 2024-12-09T08:18:12,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292657 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292665 2024-12-09T08:18:12,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,671 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,671 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=717, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,671 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292665) roll requested 2024-12-09T08:18:12,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292671 2024-12-09T08:18:12,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292665 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292671 2024-12-09T08:18:12,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,676 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,676 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=718, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,676 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292671) roll requested 2024-12-09T08:18:12,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292676 2024-12-09T08:18:12,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292671 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292676 2024-12-09T08:18:12,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,681 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,681 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=719, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,681 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292676) roll requested 2024-12-09T08:18:12,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292681 2024-12-09T08:18:12,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292676 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292681 2024-12-09T08:18:12,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,686 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,686 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=720, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,686 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292681) roll requested 2024-12-09T08:18:12,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292686 2024-12-09T08:18:12,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292681 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292686 2024-12-09T08:18:12,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,692 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,692 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=721, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,693 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292686) roll requested 2024-12-09T08:18:12,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292693 2024-12-09T08:18:12,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292686 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292693 2024-12-09T08:18:12,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,699 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,700 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=722, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,700 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292693) roll requested 2024-12-09T08:18:12,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292700 2024-12-09T08:18:12,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292693 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292700 2024-12-09T08:18:12,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,707 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,707 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=723, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,708 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292700) roll requested 2024-12-09T08:18:12,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292708 2024-12-09T08:18:12,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292700 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292708 2024-12-09T08:18:12,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,717 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,717 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=724, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,717 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292708) roll requested 2024-12-09T08:18:12,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292717 2024-12-09T08:18:12,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292708 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292717 2024-12-09T08:18:12,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,725 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,725 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=725, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,726 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292717) roll requested 2024-12-09T08:18:12,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292726 2024-12-09T08:18:12,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292717 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292726 2024-12-09T08:18:12,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,733 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,733 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=726, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,733 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292726) roll requested 2024-12-09T08:18:12,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292733 2024-12-09T08:18:12,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292726 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292733 2024-12-09T08:18:12,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,740 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,741 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=727, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,741 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292733) roll requested 2024-12-09T08:18:12,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292741 2024-12-09T08:18:12,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292733 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292741 2024-12-09T08:18:12,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,749 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,749 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=728, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,749 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292741) roll requested 2024-12-09T08:18:12,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292749 2024-12-09T08:18:12,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292741 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292749 2024-12-09T08:18:12,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,758 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,758 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=729, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,758 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292749) roll requested 2024-12-09T08:18:12,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292758 2024-12-09T08:18:12,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292749 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292758 2024-12-09T08:18:12,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,767 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,768 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=730, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,768 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292758) roll requested 2024-12-09T08:18:12,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292768 2024-12-09T08:18:12,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292758 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292768 2024-12-09T08:18:12,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,775 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,775 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=731, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,776 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292768) roll requested 2024-12-09T08:18:12,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292776 2024-12-09T08:18:12,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292768 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292776 2024-12-09T08:18:12,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,781 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,781 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=732, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,782 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292776) roll requested 2024-12-09T08:18:12,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292782 2024-12-09T08:18:12,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292776 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292782 2024-12-09T08:18:12,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,791 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,791 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=733, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,791 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292782) roll requested 2024-12-09T08:18:12,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292791 2024-12-09T08:18:12,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292782 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292791 2024-12-09T08:18:12,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,800 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,800 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=734, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,800 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292791) roll requested 2024-12-09T08:18:12,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292800 2024-12-09T08:18:12,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292791 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292800 2024-12-09T08:18:12,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,808 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,808 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=735, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,808 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292800) roll requested 2024-12-09T08:18:12,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292808 2024-12-09T08:18:12,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292800 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292808 2024-12-09T08:18:12,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,814 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,815 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=736, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,815 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292808) roll requested 2024-12-09T08:18:12,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292815 2024-12-09T08:18:12,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292808 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292815 2024-12-09T08:18:12,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,822 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,822 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=737, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,823 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292815) roll requested 2024-12-09T08:18:12,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292823 2024-12-09T08:18:12,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292815 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292823 2024-12-09T08:18:12,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,828 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,828 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=738, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,828 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292823) roll requested 2024-12-09T08:18:12,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292828 2024-12-09T08:18:12,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292823 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292828 2024-12-09T08:18:12,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,833 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,833 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=739, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,833 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292828) roll requested 2024-12-09T08:18:12,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292833 2024-12-09T08:18:12,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292828 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292833 2024-12-09T08:18:12,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,840 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,840 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=740, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,840 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292833) roll requested 2024-12-09T08:18:12,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292841 2024-12-09T08:18:12,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292833 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292841 2024-12-09T08:18:12,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,847 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,847 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=741, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,847 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292841) roll requested 2024-12-09T08:18:12,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292848 2024-12-09T08:18:12,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292841 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292848 2024-12-09T08:18:12,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,855 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,855 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=742, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,855 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292848) roll requested 2024-12-09T08:18:12,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292855 2024-12-09T08:18:12,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292848 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292855 2024-12-09T08:18:12,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,864 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,865 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=743, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,865 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292855) roll requested 2024-12-09T08:18:12,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292865 2024-12-09T08:18:12,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292855 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292865 2024-12-09T08:18:12,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,872 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,872 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=744, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,872 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292865) roll requested 2024-12-09T08:18:12,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292872 2024-12-09T08:18:12,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292865 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292872 2024-12-09T08:18:12,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,880 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,880 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=745, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,880 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292872) roll requested 2024-12-09T08:18:12,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292880 2024-12-09T08:18:12,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292872 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292880 2024-12-09T08:18:12,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,888 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,888 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=746, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,888 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292880) roll requested 2024-12-09T08:18:12,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292889 2024-12-09T08:18:12,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292880 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292889 2024-12-09T08:18:12,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,898 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,898 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=747, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,898 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292889) roll requested 2024-12-09T08:18:12,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292898 2024-12-09T08:18:12,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292889 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292898 2024-12-09T08:18:12,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,906 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,906 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=748, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,906 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292898) roll requested 2024-12-09T08:18:12,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292906 2024-12-09T08:18:12,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292898 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292906 2024-12-09T08:18:12,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,915 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,915 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=749, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,915 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292906) roll requested 2024-12-09T08:18:12,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292915 2024-12-09T08:18:12,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292906 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292915 2024-12-09T08:18:12,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,923 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,924 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=750, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,924 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292915) roll requested 2024-12-09T08:18:12,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292924 2024-12-09T08:18:12,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292915 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292924 2024-12-09T08:18:12,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,932 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,933 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=751, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,933 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292924) roll requested 2024-12-09T08:18:12,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292933 2024-12-09T08:18:12,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292924 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292933 2024-12-09T08:18:12,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,941 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,941 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=752, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,941 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292933) roll requested 2024-12-09T08:18:12,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292941 2024-12-09T08:18:12,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292933 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292941 2024-12-09T08:18:12,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,948 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,948 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=753, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,948 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292941) roll requested 2024-12-09T08:18:12,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292948 2024-12-09T08:18:12,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292941 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292948 2024-12-09T08:18:12,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,955 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,956 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=754, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,956 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292948) roll requested 2024-12-09T08:18:12,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292956 2024-12-09T08:18:12,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292948 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292956 2024-12-09T08:18:12,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,964 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,964 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=755, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,964 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292956) roll requested 2024-12-09T08:18:12,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292964 2024-12-09T08:18:12,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292956 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292964 2024-12-09T08:18:12,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,971 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,972 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=756, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,972 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292964) roll requested 2024-12-09T08:18:12,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292972 2024-12-09T08:18:12,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292964 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292972 2024-12-09T08:18:12,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,980 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,981 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=757, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,981 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292972) roll requested 2024-12-09T08:18:12,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292981 2024-12-09T08:18:12,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292972 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292981 2024-12-09T08:18:12,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,988 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,989 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=758, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,989 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292981) roll requested 2024-12-09T08:18:12,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292989 2024-12-09T08:18:12,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:12,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292981 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292989 2024-12-09T08:18:12,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:12,997 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:12,997 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=759, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:12,997 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:12,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292989) roll requested 2024-12-09T08:18:12,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732292998 2024-12-09T08:18:13,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292989 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292998 2024-12-09T08:18:13,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,004 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,005 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=760, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,005 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732292998) roll requested 2024-12-09T08:18:13,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293005 2024-12-09T08:18:13,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732292998 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293005 2024-12-09T08:18:13,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,011 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,011 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=761, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,011 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293005) roll requested 2024-12-09T08:18:13,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293011 2024-12-09T08:18:13,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293005 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293011 2024-12-09T08:18:13,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,019 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,020 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=762, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,020 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293011) roll requested 2024-12-09T08:18:13,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293020 2024-12-09T08:18:13,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293011 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293020 2024-12-09T08:18:13,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,027 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,028 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=763, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,028 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293020) roll requested 2024-12-09T08:18:13,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293028 2024-12-09T08:18:13,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293020 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293028 2024-12-09T08:18:13,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,036 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,036 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=764, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,036 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293028) roll requested 2024-12-09T08:18:13,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293036 2024-12-09T08:18:13,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293028 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293036 2024-12-09T08:18:13,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,045 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,046 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=765, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,046 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293036) roll requested 2024-12-09T08:18:13,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293046 2024-12-09T08:18:13,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293036 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293046 2024-12-09T08:18:13,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,052 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,052 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=766, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,052 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293046) roll requested 2024-12-09T08:18:13,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293052 2024-12-09T08:18:13,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293046 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293052 2024-12-09T08:18:13,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,058 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,058 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=767, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,058 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293052) roll requested 2024-12-09T08:18:13,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293059 2024-12-09T08:18:13,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293052 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293059 2024-12-09T08:18:13,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,064 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,064 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=768, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,065 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293059) roll requested 2024-12-09T08:18:13,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293065 2024-12-09T08:18:13,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293059 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293065 2024-12-09T08:18:13,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,071 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,071 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=769, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,071 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293065) roll requested 2024-12-09T08:18:13,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293072 2024-12-09T08:18:13,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293065 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293072 2024-12-09T08:18:13,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,079 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,079 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=770, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,079 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293072) roll requested 2024-12-09T08:18:13,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293079 2024-12-09T08:18:13,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293072 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293079 2024-12-09T08:18:13,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,089 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,089 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=771, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,089 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293079) roll requested 2024-12-09T08:18:13,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293089 2024-12-09T08:18:13,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293079 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293089 2024-12-09T08:18:13,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,099 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,099 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=772, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,099 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293089) roll requested 2024-12-09T08:18:13,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293099 2024-12-09T08:18:13,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293089 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293099 2024-12-09T08:18:13,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,107 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,107 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=773, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,107 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293099) roll requested 2024-12-09T08:18:13,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293107 2024-12-09T08:18:13,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293099 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293107 2024-12-09T08:18:13,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,115 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,115 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=774, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,115 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293107) roll requested 2024-12-09T08:18:13,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293115 2024-12-09T08:18:13,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293107 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293115 2024-12-09T08:18:13,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,123 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,123 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=775, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,123 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293115) roll requested 2024-12-09T08:18:13,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293123 2024-12-09T08:18:13,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293115 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293123 2024-12-09T08:18:13,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,132 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,132 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=776, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,132 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293123) roll requested 2024-12-09T08:18:13,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293132 2024-12-09T08:18:13,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293123 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293132 2024-12-09T08:18:13,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,140 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,140 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=777, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,140 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293132) roll requested 2024-12-09T08:18:13,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293140 2024-12-09T08:18:13,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293132 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293140 2024-12-09T08:18:13,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,148 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,149 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=778, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,149 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293140) roll requested 2024-12-09T08:18:13,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293149 2024-12-09T08:18:13,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293140 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293149 2024-12-09T08:18:13,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,157 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,157 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=779, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,157 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293149) roll requested 2024-12-09T08:18:13,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293158 2024-12-09T08:18:13,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293149 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293158 2024-12-09T08:18:13,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,164 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,164 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=780, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,164 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293158) roll requested 2024-12-09T08:18:13,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293164 2024-12-09T08:18:13,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293158 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293164 2024-12-09T08:18:13,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,172 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,173 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=781, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,173 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293164) roll requested 2024-12-09T08:18:13,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293173 2024-12-09T08:18:13,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293164 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293173 2024-12-09T08:18:13,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,182 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,183 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=782, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,183 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293173) roll requested 2024-12-09T08:18:13,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293183 2024-12-09T08:18:13,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293173 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293183 2024-12-09T08:18:13,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,192 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,192 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=783, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,192 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293183) roll requested 2024-12-09T08:18:13,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293192 2024-12-09T08:18:13,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293183 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293192 2024-12-09T08:18:13,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,204 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,205 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=784, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,205 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293192) roll requested 2024-12-09T08:18:13,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293205 2024-12-09T08:18:13,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293192 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293205 2024-12-09T08:18:13,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,212 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,213 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=785, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,213 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293205) roll requested 2024-12-09T08:18:13,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293213 2024-12-09T08:18:13,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293205 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293213 2024-12-09T08:18:13,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,218 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,218 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=786, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,218 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293213) roll requested 2024-12-09T08:18:13,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293218 2024-12-09T08:18:13,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293213 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293218 2024-12-09T08:18:13,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,224 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,225 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=787, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,225 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293218) roll requested 2024-12-09T08:18:13,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293225 2024-12-09T08:18:13,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293218 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293225 2024-12-09T08:18:13,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,232 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,232 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=788, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,232 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293225) roll requested 2024-12-09T08:18:13,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293232 2024-12-09T08:18:13,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293225 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293232 2024-12-09T08:18:13,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,241 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,241 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=789, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,241 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293232) roll requested 2024-12-09T08:18:13,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293241 2024-12-09T08:18:13,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293232 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293241 2024-12-09T08:18:13,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,251 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,252 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=790, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,252 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293241) roll requested 2024-12-09T08:18:13,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293252 2024-12-09T08:18:13,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293241 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293252 2024-12-09T08:18:13,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,261 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,261 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=791, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,261 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293252) roll requested 2024-12-09T08:18:13,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293262 2024-12-09T08:18:13,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293252 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293262 2024-12-09T08:18:13,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,269 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,269 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=792, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,269 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293262) roll requested 2024-12-09T08:18:13,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293269 2024-12-09T08:18:13,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293262 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293269 2024-12-09T08:18:13,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,274 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,274 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=793, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,274 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293269) roll requested 2024-12-09T08:18:13,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293274 2024-12-09T08:18:13,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293269 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293274 2024-12-09T08:18:13,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,281 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,281 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=794, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,281 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293274) roll requested 2024-12-09T08:18:13,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293281 2024-12-09T08:18:13,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293274 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293281 2024-12-09T08:18:13,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,290 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,290 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=795, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,290 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293281) roll requested 2024-12-09T08:18:13,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293290 2024-12-09T08:18:13,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293281 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293290 2024-12-09T08:18:13,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,298 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,299 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=796, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,299 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293290) roll requested 2024-12-09T08:18:13,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293299 2024-12-09T08:18:13,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293290 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293299 2024-12-09T08:18:13,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,306 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,307 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=797, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,307 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293299) roll requested 2024-12-09T08:18:13,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293307 2024-12-09T08:18:13,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293299 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293307 2024-12-09T08:18:13,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,313 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,313 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=798, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,313 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293307) roll requested 2024-12-09T08:18:13,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293314 2024-12-09T08:18:13,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293307 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293314 2024-12-09T08:18:13,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,320 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,321 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=799, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,321 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293314) roll requested 2024-12-09T08:18:13,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293321 2024-12-09T08:18:13,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293314 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293321 2024-12-09T08:18:13,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,328 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,329 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=800, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,329 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293321) roll requested 2024-12-09T08:18:13,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293329 2024-12-09T08:18:13,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293321 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293329 2024-12-09T08:18:13,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,336 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,336 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=801, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,337 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293329) roll requested 2024-12-09T08:18:13,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293337 2024-12-09T08:18:13,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293329 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293337 2024-12-09T08:18:13,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,344 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,344 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=802, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,345 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293337) roll requested 2024-12-09T08:18:13,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293345 2024-12-09T08:18:13,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293337 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293345 2024-12-09T08:18:13,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,353 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,353 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=803, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,353 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293345) roll requested 2024-12-09T08:18:13,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293353 2024-12-09T08:18:13,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293345 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293353 2024-12-09T08:18:13,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,361 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,362 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=804, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,362 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293353) roll requested 2024-12-09T08:18:13,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293362 2024-12-09T08:18:13,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293353 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293362 2024-12-09T08:18:13,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,370 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,370 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=805, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,371 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293362) roll requested 2024-12-09T08:18:13,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293371 2024-12-09T08:18:13,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293362 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293371 2024-12-09T08:18:13,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,380 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,380 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=806, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,380 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293371) roll requested 2024-12-09T08:18:13,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293380 2024-12-09T08:18:13,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293371 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293380 2024-12-09T08:18:13,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,387 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,388 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=807, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,388 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293380) roll requested 2024-12-09T08:18:13,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293388 2024-12-09T08:18:13,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293380 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293388 2024-12-09T08:18:13,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,397 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,397 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=808, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,398 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293388) roll requested 2024-12-09T08:18:13,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293398 2024-12-09T08:18:13,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293388 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293398 2024-12-09T08:18:13,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,407 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,407 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=809, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,407 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293398) roll requested 2024-12-09T08:18:13,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293407 2024-12-09T08:18:13,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293398 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293407 2024-12-09T08:18:13,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,415 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,415 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=810, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,415 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293407) roll requested 2024-12-09T08:18:13,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293415 2024-12-09T08:18:13,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293407 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293415 2024-12-09T08:18:13,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,422 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,422 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=811, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,422 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293415) roll requested 2024-12-09T08:18:13,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293422 2024-12-09T08:18:13,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293415 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293422 2024-12-09T08:18:13,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,431 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,431 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=812, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,431 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293422) roll requested 2024-12-09T08:18:13,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293431 2024-12-09T08:18:13,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293422 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293431 2024-12-09T08:18:13,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,439 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,439 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=813, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,440 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293431) roll requested 2024-12-09T08:18:13,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293440 2024-12-09T08:18:13,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293431 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293440 2024-12-09T08:18:13,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,449 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,450 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=814, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,450 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293440) roll requested 2024-12-09T08:18:13,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293450 2024-12-09T08:18:13,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293440 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293450 2024-12-09T08:18:13,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,458 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,458 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=815, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,458 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293450) roll requested 2024-12-09T08:18:13,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293458 2024-12-09T08:18:13,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293450 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293458 2024-12-09T08:18:13,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,465 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,465 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=816, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,465 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293458) roll requested 2024-12-09T08:18:13,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293466 2024-12-09T08:18:13,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293458 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293466 2024-12-09T08:18:13,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,473 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,474 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=817, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,474 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293466) roll requested 2024-12-09T08:18:13,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293474 2024-12-09T08:18:13,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293466 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293474 2024-12-09T08:18:13,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,482 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,483 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=818, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,483 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293474) roll requested 2024-12-09T08:18:13,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293483 2024-12-09T08:18:13,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293474 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293483 2024-12-09T08:18:13,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,493 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,493 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=819, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,493 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293483) roll requested 2024-12-09T08:18:13,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293493 2024-12-09T08:18:13,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293483 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293493 2024-12-09T08:18:13,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,500 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,500 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=820, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,500 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293493) roll requested 2024-12-09T08:18:13,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293501 2024-12-09T08:18:13,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293493 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293501 2024-12-09T08:18:13,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,508 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,509 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=821, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,509 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293501) roll requested 2024-12-09T08:18:13,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293509 2024-12-09T08:18:13,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293501 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293509 2024-12-09T08:18:13,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,518 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,518 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=822, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,518 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293509) roll requested 2024-12-09T08:18:13,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293518 2024-12-09T08:18:13,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293509 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293518 2024-12-09T08:18:13,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,523 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,523 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=823, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,523 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293518) roll requested 2024-12-09T08:18:13,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293523 2024-12-09T08:18:13,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293518 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293523 2024-12-09T08:18:13,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,528 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,528 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=824, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,528 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293523) roll requested 2024-12-09T08:18:13,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293528 2024-12-09T08:18:13,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293523 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293528 2024-12-09T08:18:13,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,535 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,536 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=825, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,536 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293528) roll requested 2024-12-09T08:18:13,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293536 2024-12-09T08:18:13,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293528 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293536 2024-12-09T08:18:13,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,543 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,543 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=826, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,543 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293536) roll requested 2024-12-09T08:18:13,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293543 2024-12-09T08:18:13,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293536 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293543 2024-12-09T08:18:13,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,552 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,552 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=827, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,552 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293543) roll requested 2024-12-09T08:18:13,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293552 2024-12-09T08:18:13,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293543 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293552 2024-12-09T08:18:13,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,560 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,561 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=828, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,561 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293552) roll requested 2024-12-09T08:18:13,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293562 2024-12-09T08:18:13,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293552 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293562 2024-12-09T08:18:13,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,572 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,572 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=829, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,572 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293562) roll requested 2024-12-09T08:18:13,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293572 2024-12-09T08:18:13,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293562 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293572 2024-12-09T08:18:13,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,580 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,581 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=830, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,581 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293572) roll requested 2024-12-09T08:18:13,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293581 2024-12-09T08:18:13,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293572 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293581 2024-12-09T08:18:13,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,589 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,590 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=831, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,590 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293581) roll requested 2024-12-09T08:18:13,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293590 2024-12-09T08:18:13,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293581 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293590 2024-12-09T08:18:13,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,600 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,600 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=832, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,600 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293590) roll requested 2024-12-09T08:18:13,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293600 2024-12-09T08:18:13,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293590 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293600 2024-12-09T08:18:13,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,608 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,608 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=833, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,608 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293600) roll requested 2024-12-09T08:18:13,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293609 2024-12-09T08:18:13,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293600 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293609 2024-12-09T08:18:13,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,616 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,616 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=834, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,616 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293609) roll requested 2024-12-09T08:18:13,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293617 2024-12-09T08:18:13,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293609 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293617 2024-12-09T08:18:13,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,624 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,624 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=835, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,624 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293617) roll requested 2024-12-09T08:18:13,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293624 2024-12-09T08:18:13,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293617 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293624 2024-12-09T08:18:13,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,631 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,632 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=836, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,632 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293624) roll requested 2024-12-09T08:18:13,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293632 2024-12-09T08:18:13,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293624 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293632 2024-12-09T08:18:13,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,640 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,641 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=837, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,641 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293632) roll requested 2024-12-09T08:18:13,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293641 2024-12-09T08:18:13,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293632 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293641 2024-12-09T08:18:13,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,650 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,651 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=838, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,651 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293641) roll requested 2024-12-09T08:18:13,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293651 2024-12-09T08:18:13,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293641 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293651 2024-12-09T08:18:13,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,658 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,659 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=839, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,659 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293651) roll requested 2024-12-09T08:18:13,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293659 2024-12-09T08:18:13,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293651 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293659 2024-12-09T08:18:13,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,666 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,666 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=840, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,666 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293659) roll requested 2024-12-09T08:18:13,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293666 2024-12-09T08:18:13,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293659 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293666 2024-12-09T08:18:13,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,673 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,673 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=841, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,673 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293666) roll requested 2024-12-09T08:18:13,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293673 2024-12-09T08:18:13,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293666 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293673 2024-12-09T08:18:13,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,679 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,679 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=842, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,679 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293673) roll requested 2024-12-09T08:18:13,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293680 2024-12-09T08:18:13,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293673 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293680 2024-12-09T08:18:13,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,688 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,688 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=843, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,688 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293680) roll requested 2024-12-09T08:18:13,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293688 2024-12-09T08:18:13,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293680 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293688 2024-12-09T08:18:13,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,697 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,697 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=844, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,697 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293688) roll requested 2024-12-09T08:18:13,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293697 2024-12-09T08:18:13,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293688 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293697 2024-12-09T08:18:13,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,707 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,708 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=845, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,708 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293697) roll requested 2024-12-09T08:18:13,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293708 2024-12-09T08:18:13,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293697 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293708 2024-12-09T08:18:13,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,718 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,718 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=846, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,718 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293708) roll requested 2024-12-09T08:18:13,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293719 2024-12-09T08:18:13,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293708 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293719 2024-12-09T08:18:13,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,727 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,727 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=847, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,727 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293719) roll requested 2024-12-09T08:18:13,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293727 2024-12-09T08:18:13,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293719 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293727 2024-12-09T08:18:13,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,734 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,735 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=848, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,735 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293727) roll requested 2024-12-09T08:18:13,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293735 2024-12-09T08:18:13,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293727 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293735 2024-12-09T08:18:13,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,743 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,743 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=849, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,744 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293735) roll requested 2024-12-09T08:18:13,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293744 2024-12-09T08:18:13,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293735 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293744 2024-12-09T08:18:13,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,752 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,752 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=850, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,752 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293744) roll requested 2024-12-09T08:18:13,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293753 2024-12-09T08:18:13,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293744 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293753 2024-12-09T08:18:13,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,759 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,760 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=851, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,760 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293753) roll requested 2024-12-09T08:18:13,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293760 2024-12-09T08:18:13,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293753 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293760 2024-12-09T08:18:13,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,769 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,770 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=852, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,770 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293760) roll requested 2024-12-09T08:18:13,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293770 2024-12-09T08:18:13,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293760 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293770 2024-12-09T08:18:13,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,776 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,777 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=853, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,777 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293770) roll requested 2024-12-09T08:18:13,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293777 2024-12-09T08:18:13,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293770 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293777 2024-12-09T08:18:13,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,783 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,783 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=854, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,783 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293777) roll requested 2024-12-09T08:18:13,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293783 2024-12-09T08:18:13,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293777 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293783 2024-12-09T08:18:13,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,791 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,791 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=855, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,791 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293783) roll requested 2024-12-09T08:18:13,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293791 2024-12-09T08:18:13,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293783 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293791 2024-12-09T08:18:13,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,799 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,800 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=856, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,800 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293791) roll requested 2024-12-09T08:18:13,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293800 2024-12-09T08:18:13,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293791 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293800 2024-12-09T08:18:13,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,808 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,808 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=857, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,808 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293800) roll requested 2024-12-09T08:18:13,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293808 2024-12-09T08:18:13,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293800 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293808 2024-12-09T08:18:13,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,816 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,816 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=858, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,816 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293808) roll requested 2024-12-09T08:18:13,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293817 2024-12-09T08:18:13,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293808 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293817 2024-12-09T08:18:13,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,824 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,824 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=859, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,824 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293817) roll requested 2024-12-09T08:18:13,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293824 2024-12-09T08:18:13,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293817 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293824 2024-12-09T08:18:13,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,830 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,831 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=860, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,831 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293824) roll requested 2024-12-09T08:18:13,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293831 2024-12-09T08:18:13,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293824 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293831 2024-12-09T08:18:13,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,839 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,839 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=861, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,840 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293831) roll requested 2024-12-09T08:18:13,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293840 2024-12-09T08:18:13,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293831 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293840 2024-12-09T08:18:13,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,848 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,848 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=862, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,848 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293840) roll requested 2024-12-09T08:18:13,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293849 2024-12-09T08:18:13,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293840 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293849 2024-12-09T08:18:13,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,859 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,859 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=863, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,860 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293849) roll requested 2024-12-09T08:18:13,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293860 2024-12-09T08:18:13,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293849 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293860 2024-12-09T08:18:13,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,865 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,865 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=864, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,865 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293860) roll requested 2024-12-09T08:18:13,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293865 2024-12-09T08:18:13,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293860 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293865 2024-12-09T08:18:13,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,870 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,870 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=865, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,870 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293865) roll requested 2024-12-09T08:18:13,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293870 2024-12-09T08:18:13,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293865 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293870 2024-12-09T08:18:13,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,876 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,876 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=866, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,876 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293870) roll requested 2024-12-09T08:18:13,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293876 2024-12-09T08:18:13,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293870 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293876 2024-12-09T08:18:13,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,883 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,884 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=867, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,884 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293876) roll requested 2024-12-09T08:18:13,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293884 2024-12-09T08:18:13,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293876 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293884 2024-12-09T08:18:13,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,892 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,892 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=868, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,892 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293884) roll requested 2024-12-09T08:18:13,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293893 2024-12-09T08:18:13,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293884 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293893 2024-12-09T08:18:13,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,901 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,901 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=869, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,901 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293893) roll requested 2024-12-09T08:18:13,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293901 2024-12-09T08:18:13,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293893 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293901 2024-12-09T08:18:13,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,912 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,913 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=870, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,913 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293901) roll requested 2024-12-09T08:18:13,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293913 2024-12-09T08:18:13,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293901 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293913 2024-12-09T08:18:13,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,921 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,921 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=871, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,921 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293913) roll requested 2024-12-09T08:18:13,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293921 2024-12-09T08:18:13,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293913 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293921 2024-12-09T08:18:13,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,927 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,927 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=872, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,927 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293921) roll requested 2024-12-09T08:18:13,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293927 2024-12-09T08:18:13,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293921 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293927 2024-12-09T08:18:13,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,932 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,933 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=873, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,933 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293927) roll requested 2024-12-09T08:18:13,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293933 2024-12-09T08:18:13,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293927 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293933 2024-12-09T08:18:13,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,938 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,938 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=874, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,938 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293933) roll requested 2024-12-09T08:18:13,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293938 2024-12-09T08:18:13,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293933 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293938 2024-12-09T08:18:13,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,943 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,943 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=875, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,944 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293938) roll requested 2024-12-09T08:18:13,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293944 2024-12-09T08:18:13,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293938 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293944 2024-12-09T08:18:13,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,949 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,949 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=876, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,949 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293944) roll requested 2024-12-09T08:18:13,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293949 2024-12-09T08:18:13,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293944 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293949 2024-12-09T08:18:13,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,956 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,956 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=877, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,956 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293949) roll requested 2024-12-09T08:18:13,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293956 2024-12-09T08:18:13,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293949 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293956 2024-12-09T08:18:13,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,965 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,966 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=878, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,966 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293956) roll requested 2024-12-09T08:18:13,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293966 2024-12-09T08:18:13,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293956 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293966 2024-12-09T08:18:13,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,974 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,974 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=879, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,974 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293966) roll requested 2024-12-09T08:18:13,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293975 2024-12-09T08:18:13,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293966 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293975 2024-12-09T08:18:13,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,982 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,983 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=880, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,983 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293975) roll requested 2024-12-09T08:18:13,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293983 2024-12-09T08:18:13,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293975 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293983 2024-12-09T08:18:13,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,988 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,989 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=881, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,989 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293983) roll requested 2024-12-09T08:18:13,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293989 2024-12-09T08:18:13,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:13,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293983 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293989 2024-12-09T08:18:13,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:13,995 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:13,995 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=882, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:13,995 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:13,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293989) roll requested 2024-12-09T08:18:13,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732293996 2024-12-09T08:18:14,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293989 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293996 2024-12-09T08:18:14,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,003 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,004 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=883, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,004 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732293996) roll requested 2024-12-09T08:18:14,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294004 2024-12-09T08:18:14,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732293996 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294004 2024-12-09T08:18:14,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,013 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,013 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=884, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,014 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294004) roll requested 2024-12-09T08:18:14,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294014 2024-12-09T08:18:14,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294004 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294014 2024-12-09T08:18:14,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,024 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,024 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=885, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,024 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294014) roll requested 2024-12-09T08:18:14,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294024 2024-12-09T08:18:14,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294014 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294024 2024-12-09T08:18:14,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,033 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,033 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=886, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,034 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294024) roll requested 2024-12-09T08:18:14,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294034 2024-12-09T08:18:14,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294024 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294034 2024-12-09T08:18:14,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,042 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,042 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=887, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,042 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294034) roll requested 2024-12-09T08:18:14,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294043 2024-12-09T08:18:14,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294034 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294043 2024-12-09T08:18:14,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,050 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,051 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=888, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,051 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294043) roll requested 2024-12-09T08:18:14,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294051 2024-12-09T08:18:14,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294043 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294051 2024-12-09T08:18:14,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,060 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,060 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=889, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,060 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294051) roll requested 2024-12-09T08:18:14,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294060 2024-12-09T08:18:14,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294051 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294060 2024-12-09T08:18:14,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,069 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,069 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=890, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,069 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294060) roll requested 2024-12-09T08:18:14,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294069 2024-12-09T08:18:14,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294060 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294069 2024-12-09T08:18:14,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,080 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,080 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=891, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,080 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294069) roll requested 2024-12-09T08:18:14,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294080 2024-12-09T08:18:14,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294069 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294080 2024-12-09T08:18:14,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,090 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,090 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=892, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,091 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294080) roll requested 2024-12-09T08:18:14,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294091 2024-12-09T08:18:14,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294080 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294091 2024-12-09T08:18:14,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,101 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,101 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=893, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,101 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294091) roll requested 2024-12-09T08:18:14,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294101 2024-12-09T08:18:14,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294091 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294101 2024-12-09T08:18:14,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,108 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,108 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=894, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,108 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294101) roll requested 2024-12-09T08:18:14,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294109 2024-12-09T08:18:14,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294101 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294109 2024-12-09T08:18:14,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,117 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,117 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=895, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,118 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294109) roll requested 2024-12-09T08:18:14,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294118 2024-12-09T08:18:14,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294109 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294118 2024-12-09T08:18:14,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,126 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,126 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=896, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,126 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294118) roll requested 2024-12-09T08:18:14,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294127 2024-12-09T08:18:14,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294118 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294127 2024-12-09T08:18:14,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,134 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,134 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=897, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,134 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294127) roll requested 2024-12-09T08:18:14,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294134 2024-12-09T08:18:14,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294127 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294134 2024-12-09T08:18:14,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,144 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,144 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=898, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,144 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294134) roll requested 2024-12-09T08:18:14,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294144 2024-12-09T08:18:14,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294134 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294144 2024-12-09T08:18:14,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,155 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,155 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=899, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,155 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294144) roll requested 2024-12-09T08:18:14,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294156 2024-12-09T08:18:14,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294144 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294156 2024-12-09T08:18:14,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,166 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,166 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=900, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,166 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294156) roll requested 2024-12-09T08:18:14,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294166 2024-12-09T08:18:14,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294156 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294166 2024-12-09T08:18:14,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,175 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,175 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=901, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,175 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294166) roll requested 2024-12-09T08:18:14,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294175 2024-12-09T08:18:14,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294166 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294175 2024-12-09T08:18:14,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,184 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,185 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=902, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,185 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294175) roll requested 2024-12-09T08:18:14,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294185 2024-12-09T08:18:14,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294175 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294185 2024-12-09T08:18:14,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,194 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,194 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=903, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,194 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294185) roll requested 2024-12-09T08:18:14,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294194 2024-12-09T08:18:14,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294185 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294194 2024-12-09T08:18:14,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,203 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,203 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=904, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,203 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294194) roll requested 2024-12-09T08:18:14,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294203 2024-12-09T08:18:14,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294194 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294203 2024-12-09T08:18:14,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,208 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,209 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=905, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,209 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294203) roll requested 2024-12-09T08:18:14,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294209 2024-12-09T08:18:14,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294203 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294209 2024-12-09T08:18:14,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,215 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,216 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=906, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,216 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294209) roll requested 2024-12-09T08:18:14,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294216 2024-12-09T08:18:14,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294209 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294216 2024-12-09T08:18:14,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,224 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,225 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=907, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,225 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294216) roll requested 2024-12-09T08:18:14,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294225 2024-12-09T08:18:14,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294216 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294225 2024-12-09T08:18:14,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,233 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,233 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=908, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,234 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294225) roll requested 2024-12-09T08:18:14,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294234 2024-12-09T08:18:14,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294225 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294234 2024-12-09T08:18:14,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,245 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,245 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=909, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,245 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294234) roll requested 2024-12-09T08:18:14,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294245 2024-12-09T08:18:14,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294234 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294245 2024-12-09T08:18:14,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,254 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,254 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=910, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,254 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294245) roll requested 2024-12-09T08:18:14,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294254 2024-12-09T08:18:14,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294245 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294254 2024-12-09T08:18:14,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,263 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,263 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=911, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,264 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294254) roll requested 2024-12-09T08:18:14,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294264 2024-12-09T08:18:14,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294254 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294264 2024-12-09T08:18:14,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,273 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,273 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=912, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,273 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294264) roll requested 2024-12-09T08:18:14,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294273 2024-12-09T08:18:14,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294264 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294273 2024-12-09T08:18:14,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,280 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,280 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=913, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,280 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294273) roll requested 2024-12-09T08:18:14,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294280 2024-12-09T08:18:14,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294273 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294280 2024-12-09T08:18:14,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,288 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,288 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=914, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,288 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294280) roll requested 2024-12-09T08:18:14,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294289 2024-12-09T08:18:14,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294280 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294289 2024-12-09T08:18:14,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,301 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,301 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=915, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,301 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294289) roll requested 2024-12-09T08:18:14,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294301 2024-12-09T08:18:14,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294289 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294301 2024-12-09T08:18:14,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,308 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,308 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=916, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,308 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294301) roll requested 2024-12-09T08:18:14,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294308 2024-12-09T08:18:14,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294301 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294308 2024-12-09T08:18:14,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,314 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,314 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=917, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,314 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294308) roll requested 2024-12-09T08:18:14,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294314 2024-12-09T08:18:14,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294308 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294314 2024-12-09T08:18:14,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,321 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,321 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=918, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,321 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294314) roll requested 2024-12-09T08:18:14,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294321 2024-12-09T08:18:14,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294314 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294321 2024-12-09T08:18:14,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,328 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,329 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=919, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,329 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294321) roll requested 2024-12-09T08:18:14,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294329 2024-12-09T08:18:14,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294321 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294329 2024-12-09T08:18:14,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,336 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,337 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=920, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,337 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294329) roll requested 2024-12-09T08:18:14,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294337 2024-12-09T08:18:14,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294329 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294337 2024-12-09T08:18:14,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,344 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,344 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=921, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,345 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294337) roll requested 2024-12-09T08:18:14,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294345 2024-12-09T08:18:14,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294337 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294345 2024-12-09T08:18:14,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,350 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,350 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=922, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,350 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294345) roll requested 2024-12-09T08:18:14,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294350 2024-12-09T08:18:14,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294345 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294350 2024-12-09T08:18:14,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,355 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,356 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=923, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,356 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294350) roll requested 2024-12-09T08:18:14,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294356 2024-12-09T08:18:14,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294350 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294356 2024-12-09T08:18:14,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,363 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,363 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=924, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,364 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294356) roll requested 2024-12-09T08:18:14,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294364 2024-12-09T08:18:14,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294356 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294364 2024-12-09T08:18:14,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,372 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,372 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=925, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,372 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294364) roll requested 2024-12-09T08:18:14,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294372 2024-12-09T08:18:14,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294364 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294372 2024-12-09T08:18:14,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,380 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,380 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=926, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,381 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294372) roll requested 2024-12-09T08:18:14,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294381 2024-12-09T08:18:14,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294372 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294381 2024-12-09T08:18:14,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,388 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,388 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=927, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,388 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294381) roll requested 2024-12-09T08:18:14,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294389 2024-12-09T08:18:14,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294381 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294389 2024-12-09T08:18:14,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,397 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,397 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=928, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,397 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294389) roll requested 2024-12-09T08:18:14,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294397 2024-12-09T08:18:14,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294389 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294397 2024-12-09T08:18:14,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,406 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,406 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=929, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,406 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294397) roll requested 2024-12-09T08:18:14,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294406 2024-12-09T08:18:14,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294397 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294406 2024-12-09T08:18:14,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,416 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,416 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=930, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,416 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294406) roll requested 2024-12-09T08:18:14,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294416 2024-12-09T08:18:14,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294406 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294416 2024-12-09T08:18:14,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,422 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,422 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=931, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,422 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294416) roll requested 2024-12-09T08:18:14,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294422 2024-12-09T08:18:14,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294416 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294422 2024-12-09T08:18:14,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,429 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,429 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=932, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,430 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294422) roll requested 2024-12-09T08:18:14,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294430 2024-12-09T08:18:14,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294422 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294430 2024-12-09T08:18:14,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,436 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,436 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=933, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,436 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294430) roll requested 2024-12-09T08:18:14,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294437 2024-12-09T08:18:14,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294430 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294437 2024-12-09T08:18:14,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,443 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,443 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=934, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,443 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294437) roll requested 2024-12-09T08:18:14,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294443 2024-12-09T08:18:14,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294437 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294443 2024-12-09T08:18:14,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,449 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,449 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=935, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,449 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294443) roll requested 2024-12-09T08:18:14,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294449 2024-12-09T08:18:14,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294443 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294449 2024-12-09T08:18:14,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,455 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,455 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=936, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,455 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294449) roll requested 2024-12-09T08:18:14,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294456 2024-12-09T08:18:14,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294449 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294456 2024-12-09T08:18:14,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,463 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,463 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=937, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,464 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294456) roll requested 2024-12-09T08:18:14,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294464 2024-12-09T08:18:14,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294456 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294464 2024-12-09T08:18:14,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,471 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,471 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=938, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,471 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294464) roll requested 2024-12-09T08:18:14,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294471 2024-12-09T08:18:14,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294464 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294471 2024-12-09T08:18:14,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,479 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,479 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=939, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,479 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294471) roll requested 2024-12-09T08:18:14,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294479 2024-12-09T08:18:14,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294471 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294479 2024-12-09T08:18:14,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,487 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,488 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=940, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,488 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294479) roll requested 2024-12-09T08:18:14,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294488 2024-12-09T08:18:14,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294479 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294488 2024-12-09T08:18:14,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,498 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,498 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=941, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,498 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294488) roll requested 2024-12-09T08:18:14,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294498 2024-12-09T08:18:14,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294488 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294498 2024-12-09T08:18:14,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,508 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,509 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=942, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,509 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294498) roll requested 2024-12-09T08:18:14,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294509 2024-12-09T08:18:14,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294498 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294509 2024-12-09T08:18:14,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,519 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,519 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=943, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,520 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294509) roll requested 2024-12-09T08:18:14,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294520 2024-12-09T08:18:14,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294509 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294520 2024-12-09T08:18:14,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,528 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,528 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=944, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,528 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294520) roll requested 2024-12-09T08:18:14,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294528 2024-12-09T08:18:14,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294520 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294528 2024-12-09T08:18:14,537 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,537 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=945, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,538 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294528) roll requested 2024-12-09T08:18:14,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294538 2024-12-09T08:18:14,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294528 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294538 2024-12-09T08:18:14,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,546 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,547 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=946, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,547 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294538) roll requested 2024-12-09T08:18:14,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294547 2024-12-09T08:18:14,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294538 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294547 2024-12-09T08:18:14,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,555 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,555 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=947, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,555 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294547) roll requested 2024-12-09T08:18:14,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294555 2024-12-09T08:18:14,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294547 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294555 2024-12-09T08:18:14,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,564 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,564 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=948, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,564 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294555) roll requested 2024-12-09T08:18:14,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294565 2024-12-09T08:18:14,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294555 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294565 2024-12-09T08:18:14,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,573 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,573 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=949, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,573 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294565) roll requested 2024-12-09T08:18:14,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294573 2024-12-09T08:18:14,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294565 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294573 2024-12-09T08:18:14,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,578 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,578 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=950, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,578 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294573) roll requested 2024-12-09T08:18:14,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294578 2024-12-09T08:18:14,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294573 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294578 2024-12-09T08:18:14,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,583 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,584 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=951, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,584 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294578) roll requested 2024-12-09T08:18:14,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294584 2024-12-09T08:18:14,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294578 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294584 2024-12-09T08:18:14,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,590 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,591 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=952, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,591 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294584) roll requested 2024-12-09T08:18:14,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294591 2024-12-09T08:18:14,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294584 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294591 2024-12-09T08:18:14,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,598 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,599 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=953, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,599 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294591) roll requested 2024-12-09T08:18:14,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294599 2024-12-09T08:18:14,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294591 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294599 2024-12-09T08:18:14,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,607 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,607 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=954, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,607 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294599) roll requested 2024-12-09T08:18:14,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294607 2024-12-09T08:18:14,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294599 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294607 2024-12-09T08:18:14,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,616 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,617 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=955, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,617 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294607) roll requested 2024-12-09T08:18:14,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294617 2024-12-09T08:18:14,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294607 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294617 2024-12-09T08:18:14,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,625 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,626 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=956, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,626 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294617) roll requested 2024-12-09T08:18:14,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294626 2024-12-09T08:18:14,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294617 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294626 2024-12-09T08:18:14,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,633 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,634 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=957, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,634 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294626) roll requested 2024-12-09T08:18:14,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294634 2024-12-09T08:18:14,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294626 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294634 2024-12-09T08:18:14,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,642 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,642 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=958, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,642 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294634) roll requested 2024-12-09T08:18:14,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294642 2024-12-09T08:18:14,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294634 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294642 2024-12-09T08:18:14,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,650 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,651 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=959, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,651 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294642) roll requested 2024-12-09T08:18:14,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294651 2024-12-09T08:18:14,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294642 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294651 2024-12-09T08:18:14,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,659 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,659 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=960, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,659 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294651) roll requested 2024-12-09T08:18:14,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294659 2024-12-09T08:18:14,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294651 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294659 2024-12-09T08:18:14,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,669 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,669 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=961, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,669 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294659) roll requested 2024-12-09T08:18:14,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294669 2024-12-09T08:18:14,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294659 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294669 2024-12-09T08:18:14,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,676 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,676 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=962, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,676 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294669) roll requested 2024-12-09T08:18:14,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294676 2024-12-09T08:18:14,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294669 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294676 2024-12-09T08:18:14,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,682 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,683 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=963, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,683 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294676) roll requested 2024-12-09T08:18:14,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294683 2024-12-09T08:18:14,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294676 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294683 2024-12-09T08:18:14,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,691 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,691 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=964, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,691 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294683) roll requested 2024-12-09T08:18:14,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294691 2024-12-09T08:18:14,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294683 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294691 2024-12-09T08:18:14,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,701 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,701 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=965, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,702 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294691) roll requested 2024-12-09T08:18:14,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294702 2024-12-09T08:18:14,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294691 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294702 2024-12-09T08:18:14,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,709 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,709 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=966, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,709 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294702) roll requested 2024-12-09T08:18:14,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294709 2024-12-09T08:18:14,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294702 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294709 2024-12-09T08:18:14,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,719 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,719 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=967, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,719 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294709) roll requested 2024-12-09T08:18:14,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294719 2024-12-09T08:18:14,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294719 2024-12-09T08:18:14,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,726 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,726 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=968, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,726 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294719) roll requested 2024-12-09T08:18:14,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294726 2024-12-09T08:18:14,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294719 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294726 2024-12-09T08:18:14,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,732 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,732 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=969, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,732 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294726) roll requested 2024-12-09T08:18:14,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294732 2024-12-09T08:18:14,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294726 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294732 2024-12-09T08:18:14,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,737 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,737 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=970, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,737 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294732) roll requested 2024-12-09T08:18:14,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294737 2024-12-09T08:18:14,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294732 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294737 2024-12-09T08:18:14,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,746 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,747 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=971, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,747 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294737) roll requested 2024-12-09T08:18:14,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294747 2024-12-09T08:18:14,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294737 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294747 2024-12-09T08:18:14,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,752 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,752 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=972, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,752 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294747) roll requested 2024-12-09T08:18:14,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294752 2024-12-09T08:18:14,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294747 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294752 2024-12-09T08:18:14,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,761 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,761 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=973, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,761 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294752) roll requested 2024-12-09T08:18:14,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294761 2024-12-09T08:18:14,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294752 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294761 2024-12-09T08:18:14,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,767 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,767 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=974, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,767 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294761) roll requested 2024-12-09T08:18:14,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294767 2024-12-09T08:18:14,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294761 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294767 2024-12-09T08:18:14,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,772 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,772 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=975, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,772 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294767) roll requested 2024-12-09T08:18:14,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294772 2024-12-09T08:18:14,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294767 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294772 2024-12-09T08:18:14,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,778 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,778 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=976, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,778 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294772) roll requested 2024-12-09T08:18:14,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294778 2024-12-09T08:18:14,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294772 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294778 2024-12-09T08:18:14,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,784 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,784 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=977, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,784 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294778) roll requested 2024-12-09T08:18:14,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294784 2024-12-09T08:18:14,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294778 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294784 2024-12-09T08:18:14,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,792 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,792 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=978, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,792 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294784) roll requested 2024-12-09T08:18:14,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294792 2024-12-09T08:18:14,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294784 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294792 2024-12-09T08:18:14,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,800 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,801 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=979, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,801 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294792) roll requested 2024-12-09T08:18:14,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294801 2024-12-09T08:18:14,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294792 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294801 2024-12-09T08:18:14,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,809 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,809 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=980, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,809 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294801) roll requested 2024-12-09T08:18:14,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294810 2024-12-09T08:18:14,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294801 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294810 2024-12-09T08:18:14,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,817 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,817 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=981, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,818 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294810) roll requested 2024-12-09T08:18:14,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294818 2024-12-09T08:18:14,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294810 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294818 2024-12-09T08:18:14,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,827 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,827 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=982, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,828 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294818) roll requested 2024-12-09T08:18:14,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294828 2024-12-09T08:18:14,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294818 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294828 2024-12-09T08:18:14,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,837 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,837 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=983, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,837 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294828) roll requested 2024-12-09T08:18:14,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294837 2024-12-09T08:18:14,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294828 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294837 2024-12-09T08:18:14,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,845 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,846 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=984, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,846 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294837) roll requested 2024-12-09T08:18:14,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294846 2024-12-09T08:18:14,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294837 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294846 2024-12-09T08:18:14,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,854 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,854 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=985, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,854 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294846) roll requested 2024-12-09T08:18:14,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294854 2024-12-09T08:18:14,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294846 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294854 2024-12-09T08:18:14,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,864 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,864 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=986, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,864 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294854) roll requested 2024-12-09T08:18:14,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294865 2024-12-09T08:18:14,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294854 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294865 2024-12-09T08:18:14,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,873 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,873 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=987, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,873 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294865) roll requested 2024-12-09T08:18:14,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294873 2024-12-09T08:18:14,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294865 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294873 2024-12-09T08:18:14,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,879 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,879 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=988, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,879 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294873) roll requested 2024-12-09T08:18:14,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294879 2024-12-09T08:18:14,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294873 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294879 2024-12-09T08:18:14,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,885 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,885 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=989, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,885 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294879) roll requested 2024-12-09T08:18:14,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294885 2024-12-09T08:18:14,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294879 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294885 2024-12-09T08:18:14,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,893 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,893 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=990, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,893 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294885) roll requested 2024-12-09T08:18:14,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294893 2024-12-09T08:18:14,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294885 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294893 2024-12-09T08:18:14,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,900 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,900 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=991, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,900 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294893) roll requested 2024-12-09T08:18:14,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294900 2024-12-09T08:18:14,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294893 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294900 2024-12-09T08:18:14,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,908 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,909 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=992, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,909 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294900) roll requested 2024-12-09T08:18:14,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294909 2024-12-09T08:18:14,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294900 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294909 2024-12-09T08:18:14,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,918 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,919 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=993, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,919 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294909) roll requested 2024-12-09T08:18:14,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294919 2024-12-09T08:18:14,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294909 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294919 2024-12-09T08:18:14,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,927 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,928 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=994, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,928 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294919) roll requested 2024-12-09T08:18:14,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294928 2024-12-09T08:18:14,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294919 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294928 2024-12-09T08:18:14,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,938 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,938 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=995, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,938 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294928) roll requested 2024-12-09T08:18:14,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294938 2024-12-09T08:18:14,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294928 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294938 2024-12-09T08:18:14,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,944 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,944 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=996, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,944 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294938) roll requested 2024-12-09T08:18:14,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294944 2024-12-09T08:18:14,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294938 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294944 2024-12-09T08:18:14,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,949 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,949 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=997, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,949 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294944) roll requested 2024-12-09T08:18:14,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294949 2024-12-09T08:18:14,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294944 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294949 2024-12-09T08:18:14,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,956 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,956 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=998, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,956 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294949) roll requested 2024-12-09T08:18:14,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294956 2024-12-09T08:18:14,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294949 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294956 2024-12-09T08:18:14,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,964 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,964 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=999, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,964 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294956) roll requested 2024-12-09T08:18:14,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294965 2024-12-09T08:18:14,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294956 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294965 2024-12-09T08:18:14,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,973 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,973 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1000, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,973 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294965) roll requested 2024-12-09T08:18:14,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294973 2024-12-09T08:18:14,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294965 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294973 2024-12-09T08:18:14,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,979 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,980 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1001, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,980 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294973) roll requested 2024-12-09T08:18:14,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294980 2024-12-09T08:18:14,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294973 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294980 2024-12-09T08:18:14,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,987 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,987 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1002, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,988 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294980) roll requested 2024-12-09T08:18:14,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294988 2024-12-09T08:18:14,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:14,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294980 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294988 2024-12-09T08:18:14,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:14,996 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:14,997 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1003, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:14,997 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:14,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294988) roll requested 2024-12-09T08:18:14,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732294997 2024-12-09T08:18:15,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294988 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294997 2024-12-09T08:18:15,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,005 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,005 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1004, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,005 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732294997) roll requested 2024-12-09T08:18:15,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295006 2024-12-09T08:18:15,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732294997 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295006 2024-12-09T08:18:15,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,012 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,012 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1005, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,012 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295006) roll requested 2024-12-09T08:18:15,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295012 2024-12-09T08:18:15,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295006 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295012 2024-12-09T08:18:15,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,021 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,021 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1006, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,022 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295012) roll requested 2024-12-09T08:18:15,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295022 2024-12-09T08:18:15,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295012 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295022 2024-12-09T08:18:15,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,028 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,029 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1007, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,029 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295022) roll requested 2024-12-09T08:18:15,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295029 2024-12-09T08:18:15,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295022 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295029 2024-12-09T08:18:15,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,037 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,037 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1008, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,037 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295029) roll requested 2024-12-09T08:18:15,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295037 2024-12-09T08:18:15,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295029 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295037 2024-12-09T08:18:15,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,044 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,044 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1009, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,044 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295037) roll requested 2024-12-09T08:18:15,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295044 2024-12-09T08:18:15,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295037 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295044 2024-12-09T08:18:15,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,053 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,053 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1010, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,054 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295044) roll requested 2024-12-09T08:18:15,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295054 2024-12-09T08:18:15,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295044 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295054 2024-12-09T08:18:15,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,060 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,060 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1011, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,060 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295054) roll requested 2024-12-09T08:18:15,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295060 2024-12-09T08:18:15,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295054 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295060 2024-12-09T08:18:15,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,065 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,065 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1012, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,065 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295060) roll requested 2024-12-09T08:18:15,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295065 2024-12-09T08:18:15,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295060 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295065 2024-12-09T08:18:15,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,070 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,071 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1013, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,071 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295065) roll requested 2024-12-09T08:18:15,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295071 2024-12-09T08:18:15,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295065 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295071 2024-12-09T08:18:15,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,076 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,077 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1014, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,077 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295071) roll requested 2024-12-09T08:18:15,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295077 2024-12-09T08:18:15,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295071 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295077 2024-12-09T08:18:15,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,083 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,083 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1015, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,083 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295077) roll requested 2024-12-09T08:18:15,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295083 2024-12-09T08:18:15,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295077 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295083 2024-12-09T08:18:15,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,091 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,092 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1016, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,092 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295083) roll requested 2024-12-09T08:18:15,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295092 2024-12-09T08:18:15,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295083 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295092 2024-12-09T08:18:15,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,099 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,099 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1017, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,099 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295092) roll requested 2024-12-09T08:18:15,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295100 2024-12-09T08:18:15,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295092 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295100 2024-12-09T08:18:15,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,105 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,105 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1018, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,105 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295100) roll requested 2024-12-09T08:18:15,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295105 2024-12-09T08:18:15,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295100 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295105 2024-12-09T08:18:15,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,111 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,111 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1019, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,111 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295105) roll requested 2024-12-09T08:18:15,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295112 2024-12-09T08:18:15,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295105 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295112 2024-12-09T08:18:15,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,117 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,118 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1020, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,118 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295112) roll requested 2024-12-09T08:18:15,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295118 2024-12-09T08:18:15,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295112 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295118 2024-12-09T08:18:15,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,123 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,124 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1021, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,124 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295118) roll requested 2024-12-09T08:18:15,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295124 2024-12-09T08:18:15,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295118 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295124 2024-12-09T08:18:15,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,131 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,131 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1022, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,131 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295124) roll requested 2024-12-09T08:18:15,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295131 2024-12-09T08:18:15,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295124 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295131 2024-12-09T08:18:15,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,139 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,140 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1023, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,140 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295131) roll requested 2024-12-09T08:18:15,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295140 2024-12-09T08:18:15,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295131 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295140 2024-12-09T08:18:15,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,148 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,148 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1024, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,148 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295140) roll requested 2024-12-09T08:18:15,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295149 2024-12-09T08:18:15,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295140 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295149 2024-12-09T08:18:15,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,156 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,156 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1025, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,156 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295149) roll requested 2024-12-09T08:18:15,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295156 2024-12-09T08:18:15,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295149 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295156 2024-12-09T08:18:15,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,163 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,164 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1026, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,164 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295156) roll requested 2024-12-09T08:18:15,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295164 2024-12-09T08:18:15,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295156 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295164 2024-12-09T08:18:15,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,172 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,173 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1027, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,173 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295164) roll requested 2024-12-09T08:18:15,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295173 2024-12-09T08:18:15,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295164 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295173 2024-12-09T08:18:15,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,182 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,182 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1028, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,183 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295173) roll requested 2024-12-09T08:18:15,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295183 2024-12-09T08:18:15,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295173 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295183 2024-12-09T08:18:15,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,193 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,193 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1029, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,193 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295183) roll requested 2024-12-09T08:18:15,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295193 2024-12-09T08:18:15,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295183 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295193 2024-12-09T08:18:15,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,203 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,203 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1030, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,204 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295193) roll requested 2024-12-09T08:18:15,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295204 2024-12-09T08:18:15,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295193 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295204 2024-12-09T08:18:15,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,210 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,210 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1031, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,210 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295204) roll requested 2024-12-09T08:18:15,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295210 2024-12-09T08:18:15,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295204 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295210 2024-12-09T08:18:15,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,217 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,217 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1032, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,217 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295210) roll requested 2024-12-09T08:18:15,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295217 2024-12-09T08:18:15,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295210 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295217 2024-12-09T08:18:15,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,225 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,225 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1033, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,225 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295217) roll requested 2024-12-09T08:18:15,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295226 2024-12-09T08:18:15,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295217 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295226 2024-12-09T08:18:15,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,235 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,235 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1034, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,235 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295226) roll requested 2024-12-09T08:18:15,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295236 2024-12-09T08:18:15,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295226 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295236 2024-12-09T08:18:15,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,246 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,246 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1035, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,246 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295236) roll requested 2024-12-09T08:18:15,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295246 2024-12-09T08:18:15,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295236 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295246 2024-12-09T08:18:15,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,255 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,255 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1036, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,255 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295246) roll requested 2024-12-09T08:18:15,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295255 2024-12-09T08:18:15,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295246 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295255 2024-12-09T08:18:15,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,262 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,263 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1037, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,263 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295255) roll requested 2024-12-09T08:18:15,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295263 2024-12-09T08:18:15,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295255 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295263 2024-12-09T08:18:15,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,271 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,271 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1038, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,271 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295263) roll requested 2024-12-09T08:18:15,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295271 2024-12-09T08:18:15,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295263 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295271 2024-12-09T08:18:15,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,279 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,280 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1039, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,280 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295271) roll requested 2024-12-09T08:18:15,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295280 2024-12-09T08:18:15,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295271 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295280 2024-12-09T08:18:15,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,288 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,289 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1040, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,289 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295280) roll requested 2024-12-09T08:18:15,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295289 2024-12-09T08:18:15,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295280 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295289 2024-12-09T08:18:15,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,297 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,298 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1041, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,298 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295289) roll requested 2024-12-09T08:18:15,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295298 2024-12-09T08:18:15,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295289 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295298 2024-12-09T08:18:15,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,306 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,306 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1042, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,306 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295298) roll requested 2024-12-09T08:18:15,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295307 2024-12-09T08:18:15,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295298 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295307 2024-12-09T08:18:15,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,315 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,316 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1043, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,316 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295307) roll requested 2024-12-09T08:18:15,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295316 2024-12-09T08:18:15,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295307 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295316 2024-12-09T08:18:15,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,324 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,325 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1044, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,325 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295316) roll requested 2024-12-09T08:18:15,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295325 2024-12-09T08:18:15,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295316 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295325 2024-12-09T08:18:15,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,333 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,334 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1045, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,334 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295325) roll requested 2024-12-09T08:18:15,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295334 2024-12-09T08:18:15,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295325 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295334 2024-12-09T08:18:15,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,341 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,341 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1046, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,341 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295334) roll requested 2024-12-09T08:18:15,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295341 2024-12-09T08:18:15,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295334 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295341 2024-12-09T08:18:15,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,350 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,350 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1047, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,350 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295341) roll requested 2024-12-09T08:18:15,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295350 2024-12-09T08:18:15,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295341 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295350 2024-12-09T08:18:15,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,357 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,358 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1048, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,358 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295350) roll requested 2024-12-09T08:18:15,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295358 2024-12-09T08:18:15,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295350 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295358 2024-12-09T08:18:15,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,364 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,365 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1049, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,365 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295358) roll requested 2024-12-09T08:18:15,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295365 2024-12-09T08:18:15,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295358 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295365 2024-12-09T08:18:15,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,373 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,373 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1050, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,373 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295365) roll requested 2024-12-09T08:18:15,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295373 2024-12-09T08:18:15,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295365 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295373 2024-12-09T08:18:15,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,381 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,381 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1051, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,382 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295373) roll requested 2024-12-09T08:18:15,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295382 2024-12-09T08:18:15,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295373 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295382 2024-12-09T08:18:15,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,389 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,390 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1052, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,390 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295382) roll requested 2024-12-09T08:18:15,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295390 2024-12-09T08:18:15,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295382 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295390 2024-12-09T08:18:15,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,398 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,398 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1053, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,399 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295390) roll requested 2024-12-09T08:18:15,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295399 2024-12-09T08:18:15,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,408 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295390 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295399 2024-12-09T08:18:15,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,409 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,409 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1054, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,409 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295399) roll requested 2024-12-09T08:18:15,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295409 2024-12-09T08:18:15,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295399 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295409 2024-12-09T08:18:15,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,417 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,417 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1055, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,417 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295409) roll requested 2024-12-09T08:18:15,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295417 2024-12-09T08:18:15,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295409 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295417 2024-12-09T08:18:15,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,424 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,424 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1056, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,424 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295417) roll requested 2024-12-09T08:18:15,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295425 2024-12-09T08:18:15,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295417 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295425 2024-12-09T08:18:15,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,432 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,432 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1057, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,432 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295425) roll requested 2024-12-09T08:18:15,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295432 2024-12-09T08:18:15,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295425 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295432 2024-12-09T08:18:15,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,440 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,440 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1058, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,440 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295432) roll requested 2024-12-09T08:18:15,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295440 2024-12-09T08:18:15,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295432 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295440 2024-12-09T08:18:15,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,449 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,449 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1059, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,449 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295440) roll requested 2024-12-09T08:18:15,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295449 2024-12-09T08:18:15,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295440 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295449 2024-12-09T08:18:15,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,458 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,459 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1060, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,459 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295449) roll requested 2024-12-09T08:18:15,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295459 2024-12-09T08:18:15,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295449 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295459 2024-12-09T08:18:15,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,466 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,467 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1061, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,467 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295459) roll requested 2024-12-09T08:18:15,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295467 2024-12-09T08:18:15,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295459 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295467 2024-12-09T08:18:15,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,474 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,474 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1062, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,475 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295467) roll requested 2024-12-09T08:18:15,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295475 2024-12-09T08:18:15,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295467 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295475 2024-12-09T08:18:15,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,482 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,483 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1063, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,483 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295475) roll requested 2024-12-09T08:18:15,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295483 2024-12-09T08:18:15,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295475 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295483 2024-12-09T08:18:15,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,490 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,490 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1064, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,490 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295483) roll requested 2024-12-09T08:18:15,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295490 2024-12-09T08:18:15,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295483 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295490 2024-12-09T08:18:15,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,497 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,498 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1065, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,498 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295490) roll requested 2024-12-09T08:18:15,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295498 2024-12-09T08:18:15,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295490 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295498 2024-12-09T08:18:15,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,505 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,505 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1066, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,505 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295498) roll requested 2024-12-09T08:18:15,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295506 2024-12-09T08:18:15,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295498 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295506 2024-12-09T08:18:15,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,514 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,514 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1067, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,515 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295506) roll requested 2024-12-09T08:18:15,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295515 2024-12-09T08:18:15,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295506 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295515 2024-12-09T08:18:15,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,522 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,522 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1068, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,522 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295515) roll requested 2024-12-09T08:18:15,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295522 2024-12-09T08:18:15,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295515 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295522 2024-12-09T08:18:15,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,531 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,532 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1069, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,532 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295522) roll requested 2024-12-09T08:18:15,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295532 2024-12-09T08:18:15,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295522 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295532 2024-12-09T08:18:15,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,542 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,542 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1070, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,542 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295532) roll requested 2024-12-09T08:18:15,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295542 2024-12-09T08:18:15,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295532 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295542 2024-12-09T08:18:15,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,553 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,554 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1071, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,554 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295542) roll requested 2024-12-09T08:18:15,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295554 2024-12-09T08:18:15,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295542 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295554 2024-12-09T08:18:15,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,563 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,564 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1072, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,564 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295554) roll requested 2024-12-09T08:18:15,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295564 2024-12-09T08:18:15,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295554 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295564 2024-12-09T08:18:15,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,572 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,572 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1073, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,573 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295564) roll requested 2024-12-09T08:18:15,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295573 2024-12-09T08:18:15,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295564 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295573 2024-12-09T08:18:15,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,580 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,581 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1074, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,581 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295573) roll requested 2024-12-09T08:18:15,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295581 2024-12-09T08:18:15,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295573 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295581 2024-12-09T08:18:15,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,588 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,589 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1075, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,589 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295581) roll requested 2024-12-09T08:18:15,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295589 2024-12-09T08:18:15,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295581 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295589 2024-12-09T08:18:15,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,598 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,599 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1076, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,599 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295589) roll requested 2024-12-09T08:18:15,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295599 2024-12-09T08:18:15,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295589 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295599 2024-12-09T08:18:15,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,607 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,607 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1077, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,607 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295599) roll requested 2024-12-09T08:18:15,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295608 2024-12-09T08:18:15,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295599 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295608 2024-12-09T08:18:15,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,615 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,615 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1078, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,615 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295608) roll requested 2024-12-09T08:18:15,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295615 2024-12-09T08:18:15,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295608 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295615 2024-12-09T08:18:15,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,623 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,624 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1079, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,624 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295615) roll requested 2024-12-09T08:18:15,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295624 2024-12-09T08:18:15,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295615 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295624 2024-12-09T08:18:15,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,633 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,634 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1080, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,634 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295624) roll requested 2024-12-09T08:18:15,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295634 2024-12-09T08:18:15,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295624 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295634 2024-12-09T08:18:15,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,644 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,644 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1081, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,645 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295634) roll requested 2024-12-09T08:18:15,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295645 2024-12-09T08:18:15,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295634 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295645 2024-12-09T08:18:15,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,653 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,654 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1082, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,654 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295645) roll requested 2024-12-09T08:18:15,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295654 2024-12-09T08:18:15,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295645 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295654 2024-12-09T08:18:15,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,661 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,662 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1083, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,662 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295654) roll requested 2024-12-09T08:18:15,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295662 2024-12-09T08:18:15,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295654 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295662 2024-12-09T08:18:15,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,670 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,671 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1084, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,671 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295662) roll requested 2024-12-09T08:18:15,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295671 2024-12-09T08:18:15,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295662 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295671 2024-12-09T08:18:15,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,680 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,681 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1085, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,681 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295671) roll requested 2024-12-09T08:18:15,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295681 2024-12-09T08:18:15,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295671 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295681 2024-12-09T08:18:15,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,688 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,688 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1086, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,688 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295681) roll requested 2024-12-09T08:18:15,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295688 2024-12-09T08:18:15,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295681 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295688 2024-12-09T08:18:15,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,693 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,693 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1087, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,693 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295688) roll requested 2024-12-09T08:18:15,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295694 2024-12-09T08:18:15,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295688 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295694 2024-12-09T08:18:15,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,700 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,700 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1088, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,701 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295694) roll requested 2024-12-09T08:18:15,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295701 2024-12-09T08:18:15,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295694 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295701 2024-12-09T08:18:15,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,709 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,709 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1089, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,709 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295701) roll requested 2024-12-09T08:18:15,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295709 2024-12-09T08:18:15,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295701 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295709 2024-12-09T08:18:15,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,718 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,719 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1090, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,719 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295709) roll requested 2024-12-09T08:18:15,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295719 2024-12-09T08:18:15,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295719 2024-12-09T08:18:15,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,726 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,726 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1091, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,727 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295719) roll requested 2024-12-09T08:18:15,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295727 2024-12-09T08:18:15,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295719 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295727 2024-12-09T08:18:15,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,734 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,734 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1092, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,734 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295727) roll requested 2024-12-09T08:18:15,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295735 2024-12-09T08:18:15,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295727 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295735 2024-12-09T08:18:15,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,741 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,741 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1093, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,741 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295735) roll requested 2024-12-09T08:18:15,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295741 2024-12-09T08:18:15,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295735 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295741 2024-12-09T08:18:15,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,746 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,746 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1094, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,746 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295741) roll requested 2024-12-09T08:18:15,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295746 2024-12-09T08:18:15,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295741 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295746 2024-12-09T08:18:15,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,752 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,752 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1095, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,753 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295746) roll requested 2024-12-09T08:18:15,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295753 2024-12-09T08:18:15,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295746 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295753 2024-12-09T08:18:15,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,759 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,759 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1096, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,759 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295753) roll requested 2024-12-09T08:18:15,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295759 2024-12-09T08:18:15,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295753 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295759 2024-12-09T08:18:15,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,768 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,768 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1097, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,768 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295759) roll requested 2024-12-09T08:18:15,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295768 2024-12-09T08:18:15,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295759 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295768 2024-12-09T08:18:15,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,777 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,777 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1098, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,777 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295768) roll requested 2024-12-09T08:18:15,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295777 2024-12-09T08:18:15,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295768 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295777 2024-12-09T08:18:15,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,784 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,784 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1099, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,784 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295777) roll requested 2024-12-09T08:18:15,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295784 2024-12-09T08:18:15,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295777 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295784 2024-12-09T08:18:15,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,792 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,792 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1100, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,793 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295784) roll requested 2024-12-09T08:18:15,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295793 2024-12-09T08:18:15,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295784 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295793 2024-12-09T08:18:15,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,801 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,801 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1101, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,801 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295793) roll requested 2024-12-09T08:18:15,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295801 2024-12-09T08:18:15,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295793 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295801 2024-12-09T08:18:15,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,806 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,806 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1102, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,806 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295801) roll requested 2024-12-09T08:18:15,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295806 2024-12-09T08:18:15,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295801 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295806 2024-12-09T08:18:15,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,811 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,811 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1103, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,811 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295806) roll requested 2024-12-09T08:18:15,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295811 2024-12-09T08:18:15,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295806 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295811 2024-12-09T08:18:15,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,817 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1104, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,817 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,816 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295811) roll requested 2024-12-09T08:18:15,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295817 2024-12-09T08:18:15,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295811 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295817 2024-12-09T08:18:15,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,822 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,822 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1105, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,822 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295817) roll requested 2024-12-09T08:18:15,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295822 2024-12-09T08:18:15,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295817 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295822 2024-12-09T08:18:15,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,828 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,829 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1106, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,829 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295822) roll requested 2024-12-09T08:18:15,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295829 2024-12-09T08:18:15,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295822 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295829 2024-12-09T08:18:15,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,835 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,835 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1107, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,835 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295829) roll requested 2024-12-09T08:18:15,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295836 2024-12-09T08:18:15,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295829 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295836 2024-12-09T08:18:15,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,844 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,845 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1108, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,845 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295836) roll requested 2024-12-09T08:18:15,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295845 2024-12-09T08:18:15,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295836 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295845 2024-12-09T08:18:15,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,853 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,854 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1109, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,854 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295845) roll requested 2024-12-09T08:18:15,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295854 2024-12-09T08:18:15,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295845 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295854 2024-12-09T08:18:15,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,865 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,865 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1110, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,865 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295854) roll requested 2024-12-09T08:18:15,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295865 2024-12-09T08:18:15,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295854 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295865 2024-12-09T08:18:15,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,873 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,873 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1111, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,874 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295865) roll requested 2024-12-09T08:18:15,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295874 2024-12-09T08:18:15,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295865 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295874 2024-12-09T08:18:15,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,879 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,880 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1112, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,880 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295874) roll requested 2024-12-09T08:18:15,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295880 2024-12-09T08:18:15,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295874 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295880 2024-12-09T08:18:15,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,886 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,886 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1113, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,886 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295880) roll requested 2024-12-09T08:18:15,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295886 2024-12-09T08:18:15,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295880 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295886 2024-12-09T08:18:15,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,893 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,894 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1114, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,894 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295886) roll requested 2024-12-09T08:18:15,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295894 2024-12-09T08:18:15,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295886 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295894 2024-12-09T08:18:15,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,901 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,901 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1115, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,901 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295894) roll requested 2024-12-09T08:18:15,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295901 2024-12-09T08:18:15,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295894 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295901 2024-12-09T08:18:15,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,909 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,909 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1116, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,909 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295901) roll requested 2024-12-09T08:18:15,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295909 2024-12-09T08:18:15,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295901 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295909 2024-12-09T08:18:15,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,917 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,917 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1117, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,917 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295909) roll requested 2024-12-09T08:18:15,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295917 2024-12-09T08:18:15,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295909 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295917 2024-12-09T08:18:15,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,924 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,924 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1118, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,924 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295917) roll requested 2024-12-09T08:18:15,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295924 2024-12-09T08:18:15,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295917 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295924 2024-12-09T08:18:15,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,932 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,932 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1119, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,932 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295924) roll requested 2024-12-09T08:18:15,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295932 2024-12-09T08:18:15,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295924 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295932 2024-12-09T08:18:15,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,941 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,941 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1120, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,941 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295932) roll requested 2024-12-09T08:18:15,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295941 2024-12-09T08:18:15,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295932 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295941 2024-12-09T08:18:15,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,948 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,949 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1121, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,949 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295941) roll requested 2024-12-09T08:18:15,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295949 2024-12-09T08:18:15,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295941 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295949 2024-12-09T08:18:15,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,956 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,957 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1122, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,957 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295949) roll requested 2024-12-09T08:18:15,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295957 2024-12-09T08:18:15,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295949 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295957 2024-12-09T08:18:15,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,965 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,965 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1123, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,965 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295957) roll requested 2024-12-09T08:18:15,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295965 2024-12-09T08:18:15,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295957 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295965 2024-12-09T08:18:15,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,971 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,972 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1124, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,972 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295965) roll requested 2024-12-09T08:18:15,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295972 2024-12-09T08:18:15,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295965 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295972 2024-12-09T08:18:15,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,978 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,978 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1125, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,979 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295972) roll requested 2024-12-09T08:18:15,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295979 2024-12-09T08:18:15,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295972 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295979 2024-12-09T08:18:15,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,985 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,985 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1126, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,985 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295979) roll requested 2024-12-09T08:18:15,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295985 2024-12-09T08:18:15,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:15,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295979 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295985 2024-12-09T08:18:15,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:15,993 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:15,993 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1127, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:15,993 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:15,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295985) roll requested 2024-12-09T08:18:15,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732295993 2024-12-09T08:18:16,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295985 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295993 2024-12-09T08:18:16,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,001 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,001 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1128, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,002 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732295993) roll requested 2024-12-09T08:18:16,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296002 2024-12-09T08:18:16,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732295993 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296002 2024-12-09T08:18:16,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,008 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,009 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1129, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,009 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296002) roll requested 2024-12-09T08:18:16,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296009 2024-12-09T08:18:16,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296002 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296009 2024-12-09T08:18:16,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,016 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,016 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1130, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,016 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296009) roll requested 2024-12-09T08:18:16,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296016 2024-12-09T08:18:16,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296009 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296016 2024-12-09T08:18:16,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,024 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,024 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1131, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,024 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296016) roll requested 2024-12-09T08:18:16,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296024 2024-12-09T08:18:16,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296016 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296024 2024-12-09T08:18:16,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,033 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,033 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1132, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,033 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296024) roll requested 2024-12-09T08:18:16,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296034 2024-12-09T08:18:16,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296024 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296034 2024-12-09T08:18:16,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,042 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,042 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1133, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,042 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296034) roll requested 2024-12-09T08:18:16,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296042 2024-12-09T08:18:16,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296034 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296042 2024-12-09T08:18:16,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,049 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,050 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1134, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,050 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296042) roll requested 2024-12-09T08:18:16,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296050 2024-12-09T08:18:16,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296042 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296050 2024-12-09T08:18:16,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,059 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,060 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1135, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,060 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296050) roll requested 2024-12-09T08:18:16,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296060 2024-12-09T08:18:16,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296050 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296060 2024-12-09T08:18:16,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,069 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,070 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1136, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,070 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296060) roll requested 2024-12-09T08:18:16,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296070 2024-12-09T08:18:16,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296060 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296070 2024-12-09T08:18:16,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,078 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,078 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1137, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,078 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296070) roll requested 2024-12-09T08:18:16,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296078 2024-12-09T08:18:16,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296070 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296078 2024-12-09T08:18:16,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,085 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,086 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1138, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,086 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296078) roll requested 2024-12-09T08:18:16,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296086 2024-12-09T08:18:16,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296078 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296086 2024-12-09T08:18:16,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,096 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,097 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1139, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,097 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296086) roll requested 2024-12-09T08:18:16,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296097 2024-12-09T08:18:16,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296086 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296097 2024-12-09T08:18:16,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,103 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,104 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1140, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,104 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296097) roll requested 2024-12-09T08:18:16,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296104 2024-12-09T08:18:16,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296097 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296104 2024-12-09T08:18:16,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,112 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,112 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1141, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,112 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296104) roll requested 2024-12-09T08:18:16,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296112 2024-12-09T08:18:16,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296104 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296112 2024-12-09T08:18:16,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,118 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,119 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1142, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,119 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296112) roll requested 2024-12-09T08:18:16,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296119 2024-12-09T08:18:16,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296112 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296119 2024-12-09T08:18:16,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,125 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,126 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1143, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,126 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296119) roll requested 2024-12-09T08:18:16,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296126 2024-12-09T08:18:16,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296119 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296126 2024-12-09T08:18:16,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,133 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,133 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1144, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,133 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296126) roll requested 2024-12-09T08:18:16,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296133 2024-12-09T08:18:16,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296126 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296133 2024-12-09T08:18:16,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,141 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,142 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1145, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,142 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296133) roll requested 2024-12-09T08:18:16,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296142 2024-12-09T08:18:16,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296133 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296142 2024-12-09T08:18:16,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,149 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,149 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1146, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,149 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296142) roll requested 2024-12-09T08:18:16,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296149 2024-12-09T08:18:16,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296142 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296149 2024-12-09T08:18:16,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,155 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,156 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1147, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,156 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296149) roll requested 2024-12-09T08:18:16,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296156 2024-12-09T08:18:16,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296149 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296156 2024-12-09T08:18:16,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,161 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,161 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1148, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,161 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296156) roll requested 2024-12-09T08:18:16,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296161 2024-12-09T08:18:16,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296156 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296161 2024-12-09T08:18:16,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,166 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,167 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1149, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,167 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296161) roll requested 2024-12-09T08:18:16,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296167 2024-12-09T08:18:16,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296161 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296167 2024-12-09T08:18:16,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,173 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,174 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1150, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,174 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296167) roll requested 2024-12-09T08:18:16,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296174 2024-12-09T08:18:16,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296167 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296174 2024-12-09T08:18:16,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,182 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,182 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1151, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,183 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296174) roll requested 2024-12-09T08:18:16,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296183 2024-12-09T08:18:16,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296174 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296183 2024-12-09T08:18:16,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,191 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,191 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1152, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,191 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296183) roll requested 2024-12-09T08:18:16,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296191 2024-12-09T08:18:16,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296183 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296191 2024-12-09T08:18:16,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,199 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,200 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1153, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,200 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296191) roll requested 2024-12-09T08:18:16,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296200 2024-12-09T08:18:16,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296191 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296200 2024-12-09T08:18:16,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,209 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,210 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1154, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,210 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296200) roll requested 2024-12-09T08:18:16,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296210 2024-12-09T08:18:16,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296200 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296210 2024-12-09T08:18:16,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,218 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,219 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1155, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,219 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296210) roll requested 2024-12-09T08:18:16,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296219 2024-12-09T08:18:16,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296210 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296219 2024-12-09T08:18:16,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,233 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,233 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1156, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,233 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296219) roll requested 2024-12-09T08:18:16,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296233 2024-12-09T08:18:16,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296219 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296233 2024-12-09T08:18:16,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,239 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,239 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1157, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,239 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296233) roll requested 2024-12-09T08:18:16,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296239 2024-12-09T08:18:16,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296233 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296239 2024-12-09T08:18:16,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,245 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,245 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1158, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,245 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296239) roll requested 2024-12-09T08:18:16,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296245 2024-12-09T08:18:16,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296239 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296245 2024-12-09T08:18:16,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,253 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,253 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1159, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,253 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296245) roll requested 2024-12-09T08:18:16,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296253 2024-12-09T08:18:16,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296245 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296253 2024-12-09T08:18:16,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,258 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,259 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1160, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,259 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296253) roll requested 2024-12-09T08:18:16,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296259 2024-12-09T08:18:16,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296253 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296259 2024-12-09T08:18:16,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,264 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,264 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1161, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,264 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296259) roll requested 2024-12-09T08:18:16,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296264 2024-12-09T08:18:16,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296259 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296264 2024-12-09T08:18:16,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,270 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,270 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1162, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,270 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296264) roll requested 2024-12-09T08:18:16,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296270 2024-12-09T08:18:16,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296264 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296270 2024-12-09T08:18:16,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,277 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,277 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1163, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,277 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296270) roll requested 2024-12-09T08:18:16,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296277 2024-12-09T08:18:16,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296270 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296277 2024-12-09T08:18:16,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,283 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,283 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1164, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,283 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296277) roll requested 2024-12-09T08:18:16,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296283 2024-12-09T08:18:16,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296277 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296283 2024-12-09T08:18:16,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,289 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,289 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1165, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,289 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296283) roll requested 2024-12-09T08:18:16,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296289 2024-12-09T08:18:16,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296283 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296289 2024-12-09T08:18:16,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,295 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,295 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1166, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,295 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296289) roll requested 2024-12-09T08:18:16,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296295 2024-12-09T08:18:16,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296289 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296295 2024-12-09T08:18:16,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,302 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,302 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1167, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,302 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296295) roll requested 2024-12-09T08:18:16,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296302 2024-12-09T08:18:16,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296295 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296302 2024-12-09T08:18:16,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,308 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,308 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1168, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,308 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296302) roll requested 2024-12-09T08:18:16,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296308 2024-12-09T08:18:16,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296302 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296308 2024-12-09T08:18:16,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,314 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,315 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1169, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,315 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296308) roll requested 2024-12-09T08:18:16,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296315 2024-12-09T08:18:16,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296308 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296315 2024-12-09T08:18:16,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,320 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,320 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1170, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,320 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296315) roll requested 2024-12-09T08:18:16,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296320 2024-12-09T08:18:16,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296315 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296320 2024-12-09T08:18:16,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,326 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,326 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1171, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,326 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296320) roll requested 2024-12-09T08:18:16,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296326 2024-12-09T08:18:16,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296320 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296326 2024-12-09T08:18:16,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,332 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,332 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1172, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,332 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296326) roll requested 2024-12-09T08:18:16,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296332 2024-12-09T08:18:16,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296326 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296332 2024-12-09T08:18:16,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,339 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,339 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1173, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,339 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296332) roll requested 2024-12-09T08:18:16,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296339 2024-12-09T08:18:16,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296332 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296339 2024-12-09T08:18:16,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,347 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,347 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1174, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,347 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296339) roll requested 2024-12-09T08:18:16,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296347 2024-12-09T08:18:16,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296339 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296347 2024-12-09T08:18:16,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,355 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,355 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1175, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,355 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296347) roll requested 2024-12-09T08:18:16,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296355 2024-12-09T08:18:16,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296347 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296355 2024-12-09T08:18:16,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,363 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,363 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1176, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,364 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296355) roll requested 2024-12-09T08:18:16,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296364 2024-12-09T08:18:16,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296355 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296364 2024-12-09T08:18:16,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,372 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,372 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1177, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,373 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296364) roll requested 2024-12-09T08:18:16,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296373 2024-12-09T08:18:16,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296364 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296373 2024-12-09T08:18:16,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,380 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,380 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1178, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,380 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296373) roll requested 2024-12-09T08:18:16,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296380 2024-12-09T08:18:16,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296373 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296380 2024-12-09T08:18:16,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,387 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,388 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1179, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,388 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296380) roll requested 2024-12-09T08:18:16,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296388 2024-12-09T08:18:16,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296380 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296388 2024-12-09T08:18:16,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,395 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,396 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1180, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,396 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296388) roll requested 2024-12-09T08:18:16,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296396 2024-12-09T08:18:16,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296388 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296396 2024-12-09T08:18:16,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,455 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,455 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1181, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,456 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296396) roll requested 2024-12-09T08:18:16,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296456 2024-12-09T08:18:16,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296396 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296456 2024-12-09T08:18:16,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,463 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,464 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1182, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,464 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296456) roll requested 2024-12-09T08:18:16,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296464 2024-12-09T08:18:16,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296456 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296464 2024-12-09T08:18:16,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,471 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,471 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1183, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,471 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296464) roll requested 2024-12-09T08:18:16,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296471 2024-12-09T08:18:16,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296464 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296471 2024-12-09T08:18:16,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,476 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,476 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1184, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,477 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296471) roll requested 2024-12-09T08:18:16,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296477 2024-12-09T08:18:16,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296471 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296477 2024-12-09T08:18:16,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,482 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,482 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1185, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,482 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296477) roll requested 2024-12-09T08:18:16,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296482 2024-12-09T08:18:16,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296477 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296482 2024-12-09T08:18:16,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,487 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,488 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1186, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,488 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296482) roll requested 2024-12-09T08:18:16,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296488 2024-12-09T08:18:16,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296482 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296488 2024-12-09T08:18:16,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,492 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,493 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1187, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,493 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296488) roll requested 2024-12-09T08:18:16,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296493 2024-12-09T08:18:16,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296488 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296493 2024-12-09T08:18:16,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,499 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,499 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1188, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,499 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296493) roll requested 2024-12-09T08:18:16,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296499 2024-12-09T08:18:16,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296493 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296499 2024-12-09T08:18:16,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,503 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,504 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1189, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,504 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296499) roll requested 2024-12-09T08:18:16,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296504 2024-12-09T08:18:16,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296499 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296504 2024-12-09T08:18:16,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,511 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,512 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1190, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,512 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296504) roll requested 2024-12-09T08:18:16,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296512 2024-12-09T08:18:16,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296504 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296512 2024-12-09T08:18:16,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,520 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,521 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1191, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,521 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296512) roll requested 2024-12-09T08:18:16,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296521 2024-12-09T08:18:16,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296512 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296521 2024-12-09T08:18:16,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,527 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,527 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1192, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,527 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296521) roll requested 2024-12-09T08:18:16,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296527 2024-12-09T08:18:16,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296521 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296527 2024-12-09T08:18:16,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,534 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,534 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1193, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,535 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296527) roll requested 2024-12-09T08:18:16,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296535 2024-12-09T08:18:16,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296527 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296535 2024-12-09T08:18:16,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,542 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,543 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1194, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,543 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296535) roll requested 2024-12-09T08:18:16,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296543 2024-12-09T08:18:16,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296535 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296543 2024-12-09T08:18:16,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,551 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,551 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1195, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,551 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296543) roll requested 2024-12-09T08:18:16,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296551 2024-12-09T08:18:16,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296543 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296551 2024-12-09T08:18:16,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,559 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,559 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1196, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,559 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296551) roll requested 2024-12-09T08:18:16,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296559 2024-12-09T08:18:16,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296551 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296559 2024-12-09T08:18:16,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,566 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,566 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1197, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,566 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296559) roll requested 2024-12-09T08:18:16,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296566 2024-12-09T08:18:16,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296559 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296566 2024-12-09T08:18:16,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,573 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,573 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1198, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,573 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296566) roll requested 2024-12-09T08:18:16,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296573 2024-12-09T08:18:16,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296566 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296573 2024-12-09T08:18:16,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,581 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,581 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1199, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,581 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296573) roll requested 2024-12-09T08:18:16,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296581 2024-12-09T08:18:16,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296573 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296581 2024-12-09T08:18:16,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,589 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,589 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1200, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,589 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296581) roll requested 2024-12-09T08:18:16,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296589 2024-12-09T08:18:16,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296581 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296589 2024-12-09T08:18:16,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,596 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,597 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1201, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,597 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296589) roll requested 2024-12-09T08:18:16,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296597 2024-12-09T08:18:16,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296589 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296597 2024-12-09T08:18:16,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,604 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,604 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1202, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,604 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296597) roll requested 2024-12-09T08:18:16,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296604 2024-12-09T08:18:16,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296597 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296604 2024-12-09T08:18:16,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,610 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,611 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1203, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,611 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296604) roll requested 2024-12-09T08:18:16,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296611 2024-12-09T08:18:16,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296604 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296611 2024-12-09T08:18:16,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,618 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,619 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1204, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,619 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296611) roll requested 2024-12-09T08:18:16,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296619 2024-12-09T08:18:16,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296611 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296619 2024-12-09T08:18:16,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,627 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,628 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1205, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,628 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296619) roll requested 2024-12-09T08:18:16,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296628 2024-12-09T08:18:16,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296619 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296628 2024-12-09T08:18:16,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,636 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,637 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1206, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,637 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296628) roll requested 2024-12-09T08:18:16,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296637 2024-12-09T08:18:16,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296628 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296637 2024-12-09T08:18:16,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,646 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,647 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1207, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,647 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296637) roll requested 2024-12-09T08:18:16,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296647 2024-12-09T08:18:16,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296637 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296647 2024-12-09T08:18:16,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,654 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,655 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1208, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,655 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296647) roll requested 2024-12-09T08:18:16,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296655 2024-12-09T08:18:16,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296647 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296655 2024-12-09T08:18:16,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,660 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,660 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1209, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,661 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296655) roll requested 2024-12-09T08:18:16,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296661 2024-12-09T08:18:16,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296655 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296661 2024-12-09T08:18:16,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,668 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,669 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1210, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,669 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296661) roll requested 2024-12-09T08:18:16,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296669 2024-12-09T08:18:16,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296661 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296669 2024-12-09T08:18:16,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,677 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,678 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1211, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,678 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296669) roll requested 2024-12-09T08:18:16,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296678 2024-12-09T08:18:16,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296669 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296678 2024-12-09T08:18:16,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,687 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,688 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1212, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,688 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296678) roll requested 2024-12-09T08:18:16,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296688 2024-12-09T08:18:16,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296678 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296688 2024-12-09T08:18:16,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,696 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,697 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1213, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,697 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296688) roll requested 2024-12-09T08:18:16,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296697 2024-12-09T08:18:16,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296688 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296697 2024-12-09T08:18:16,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,704 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,705 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1214, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,705 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296697) roll requested 2024-12-09T08:18:16,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296705 2024-12-09T08:18:16,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296697 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296705 2024-12-09T08:18:16,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,713 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,714 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1215, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,714 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296705) roll requested 2024-12-09T08:18:16,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296714 2024-12-09T08:18:16,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296705 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296714 2024-12-09T08:18:16,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,721 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,721 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1216, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,721 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296714) roll requested 2024-12-09T08:18:16,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296721 2024-12-09T08:18:16,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296714 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296721 2024-12-09T08:18:16,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,728 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,728 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1217, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,728 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296721) roll requested 2024-12-09T08:18:16,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296729 2024-12-09T08:18:16,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296721 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296729 2024-12-09T08:18:16,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,735 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,736 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1218, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,736 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296729) roll requested 2024-12-09T08:18:16,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296736 2024-12-09T08:18:16,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296729 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296736 2024-12-09T08:18:16,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,761 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,761 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1219, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,761 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296736) roll requested 2024-12-09T08:18:16,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296761 2024-12-09T08:18:16,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296736 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296761 2024-12-09T08:18:16,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,771 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,772 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1220, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,772 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296761) roll requested 2024-12-09T08:18:16,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296772 2024-12-09T08:18:16,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296761 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296772 2024-12-09T08:18:16,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,778 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,779 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1221, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,779 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296772) roll requested 2024-12-09T08:18:16,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296779 2024-12-09T08:18:16,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296772 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296779 2024-12-09T08:18:16,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,786 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,786 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1222, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,786 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296779) roll requested 2024-12-09T08:18:16,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296786 2024-12-09T08:18:16,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296779 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296786 2024-12-09T08:18:16,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,791 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,792 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1223, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,792 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296786) roll requested 2024-12-09T08:18:16,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296792 2024-12-09T08:18:16,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296786 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296792 2024-12-09T08:18:16,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,798 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,798 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1224, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,798 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296792) roll requested 2024-12-09T08:18:16,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296798 2024-12-09T08:18:16,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296792 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296798 2024-12-09T08:18:16,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,805 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,805 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1225, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,805 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296798) roll requested 2024-12-09T08:18:16,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296805 2024-12-09T08:18:16,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296798 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296805 2024-12-09T08:18:16,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,813 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,813 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1226, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,813 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296805) roll requested 2024-12-09T08:18:16,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296813 2024-12-09T08:18:16,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296805 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296813 2024-12-09T08:18:16,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,818 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,819 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1227, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,819 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296813) roll requested 2024-12-09T08:18:16,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296819 2024-12-09T08:18:16,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296813 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296819 2024-12-09T08:18:16,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,824 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,824 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1228, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,824 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296819) roll requested 2024-12-09T08:18:16,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296824 2024-12-09T08:18:16,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296819 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296824 2024-12-09T08:18:16,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,829 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,829 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1229, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,829 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296824) roll requested 2024-12-09T08:18:16,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296830 2024-12-09T08:18:16,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296824 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296830 2024-12-09T08:18:16,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,835 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,836 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1230, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,836 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296830) roll requested 2024-12-09T08:18:16,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296836 2024-12-09T08:18:16,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296830 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296836 2024-12-09T08:18:16,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,842 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,843 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1231, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,843 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296836) roll requested 2024-12-09T08:18:16,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296843 2024-12-09T08:18:16,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296836 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296843 2024-12-09T08:18:16,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,851 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,851 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1232, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,851 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296843) roll requested 2024-12-09T08:18:16,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296851 2024-12-09T08:18:16,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296843 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296851 2024-12-09T08:18:16,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,858 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,858 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1233, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,858 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296851) roll requested 2024-12-09T08:18:16,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296859 2024-12-09T08:18:16,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296851 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296859 2024-12-09T08:18:16,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,865 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,866 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1234, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,866 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296859) roll requested 2024-12-09T08:18:16,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296866 2024-12-09T08:18:16,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296859 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296866 2024-12-09T08:18:16,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,874 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,874 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1235, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,874 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296866) roll requested 2024-12-09T08:18:16,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296874 2024-12-09T08:18:16,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296866 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296874 2024-12-09T08:18:16,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,883 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,883 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1236, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,883 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296874) roll requested 2024-12-09T08:18:16,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296884 2024-12-09T08:18:16,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296874 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296884 2024-12-09T08:18:16,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,893 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,893 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1237, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,893 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296884) roll requested 2024-12-09T08:18:16,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296893 2024-12-09T08:18:16,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296884 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296893 2024-12-09T08:18:16,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,903 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,903 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1238, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,903 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296893) roll requested 2024-12-09T08:18:16,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296903 2024-12-09T08:18:16,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296893 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296903 2024-12-09T08:18:16,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,911 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,911 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1239, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,911 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296903) roll requested 2024-12-09T08:18:16,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296911 2024-12-09T08:18:16,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296903 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296911 2024-12-09T08:18:16,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,917 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,918 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1240, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,918 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296911) roll requested 2024-12-09T08:18:16,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296918 2024-12-09T08:18:16,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296911 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296918 2024-12-09T08:18:16,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,926 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,927 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1241, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,927 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296918) roll requested 2024-12-09T08:18:16,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296927 2024-12-09T08:18:16,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296918 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296927 2024-12-09T08:18:16,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,934 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,935 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1242, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,935 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296927) roll requested 2024-12-09T08:18:16,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296935 2024-12-09T08:18:16,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296927 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296935 2024-12-09T08:18:16,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,944 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,945 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1243, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,945 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296935) roll requested 2024-12-09T08:18:16,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296945 2024-12-09T08:18:16,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296935 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296945 2024-12-09T08:18:16,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,955 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,955 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1244, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,955 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296945) roll requested 2024-12-09T08:18:16,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296955 2024-12-09T08:18:16,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296945 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296955 2024-12-09T08:18:16,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,986 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,986 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1245, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,986 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296955) roll requested 2024-12-09T08:18:16,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296987 2024-12-09T08:18:16,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:16,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296955 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296987 2024-12-09T08:18:16,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:16,997 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:16,997 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1246, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:16,997 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:16,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296987) roll requested 2024-12-09T08:18:16,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732296997 2024-12-09T08:18:17,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296987 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296997 2024-12-09T08:18:17,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,005 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,006 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1247, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,006 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732296997) roll requested 2024-12-09T08:18:17,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297006 2024-12-09T08:18:17,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732296997 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297006 2024-12-09T08:18:17,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,017 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,017 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1248, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,017 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297006) roll requested 2024-12-09T08:18:17,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297017 2024-12-09T08:18:17,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297006 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297017 2024-12-09T08:18:17,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,025 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,025 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1249, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,025 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297017) roll requested 2024-12-09T08:18:17,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297026 2024-12-09T08:18:17,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297017 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297026 2024-12-09T08:18:17,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,032 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,033 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1250, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,033 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297026) roll requested 2024-12-09T08:18:17,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297033 2024-12-09T08:18:17,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297026 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297033 2024-12-09T08:18:17,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,039 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,040 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1251, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,040 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297033) roll requested 2024-12-09T08:18:17,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297040 2024-12-09T08:18:17,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297033 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297040 2024-12-09T08:18:17,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,046 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,046 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1252, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,046 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297040) roll requested 2024-12-09T08:18:17,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297046 2024-12-09T08:18:17,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297040 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297046 2024-12-09T08:18:17,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,052 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,053 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1253, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,053 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297046) roll requested 2024-12-09T08:18:17,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297053 2024-12-09T08:18:17,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297046 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297053 2024-12-09T08:18:17,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,060 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,060 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1254, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,061 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297053) roll requested 2024-12-09T08:18:17,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297061 2024-12-09T08:18:17,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297053 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297061 2024-12-09T08:18:17,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,069 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,069 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1255, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,069 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297061) roll requested 2024-12-09T08:18:17,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297069 2024-12-09T08:18:17,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297061 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297069 2024-12-09T08:18:17,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,079 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,080 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1256, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,080 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297069) roll requested 2024-12-09T08:18:17,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297080 2024-12-09T08:18:17,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297069 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297080 2024-12-09T08:18:17,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,089 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,090 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1257, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,090 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297080) roll requested 2024-12-09T08:18:17,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297090 2024-12-09T08:18:17,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297080 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297090 2024-12-09T08:18:17,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,098 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,098 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1258, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,098 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297090) roll requested 2024-12-09T08:18:17,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297098 2024-12-09T08:18:17,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297090 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297098 2024-12-09T08:18:17,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,107 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,107 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1259, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,107 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297098) roll requested 2024-12-09T08:18:17,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297107 2024-12-09T08:18:17,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297098 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297107 2024-12-09T08:18:17,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,116 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,116 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1260, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,117 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297107) roll requested 2024-12-09T08:18:17,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297117 2024-12-09T08:18:17,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297107 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297117 2024-12-09T08:18:17,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,124 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,125 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1261, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,125 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297117) roll requested 2024-12-09T08:18:17,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297125 2024-12-09T08:18:17,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297117 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297125 2024-12-09T08:18:17,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,134 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,134 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1262, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,134 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297125) roll requested 2024-12-09T08:18:17,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297134 2024-12-09T08:18:17,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297125 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297134 2024-12-09T08:18:17,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,143 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,144 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1263, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,144 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297134) roll requested 2024-12-09T08:18:17,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297144 2024-12-09T08:18:17,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297134 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297144 2024-12-09T08:18:17,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,152 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,153 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1264, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,153 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297144) roll requested 2024-12-09T08:18:17,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297153 2024-12-09T08:18:17,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297144 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297153 2024-12-09T08:18:17,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,161 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,161 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1265, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,162 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297153) roll requested 2024-12-09T08:18:17,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297162 2024-12-09T08:18:17,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297153 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297162 2024-12-09T08:18:17,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,172 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,173 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1266, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,173 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297162) roll requested 2024-12-09T08:18:17,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297173 2024-12-09T08:18:17,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297162 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297173 2024-12-09T08:18:17,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,181 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,181 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1267, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,182 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297173) roll requested 2024-12-09T08:18:17,182 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297182 2024-12-09T08:18:17,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297173 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297182 2024-12-09T08:18:17,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,188 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,188 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1268, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,189 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297182) roll requested 2024-12-09T08:18:17,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297189 2024-12-09T08:18:17,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297182 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297189 2024-12-09T08:18:17,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,196 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,196 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1269, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,197 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297189) roll requested 2024-12-09T08:18:17,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297197 2024-12-09T08:18:17,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297189 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297197 2024-12-09T08:18:17,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,204 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,204 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1270, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,204 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297197) roll requested 2024-12-09T08:18:17,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297204 2024-12-09T08:18:17,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297197 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297204 2024-12-09T08:18:17,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,213 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,214 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1271, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,214 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297204) roll requested 2024-12-09T08:18:17,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297214 2024-12-09T08:18:17,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297204 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297214 2024-12-09T08:18:17,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,223 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,224 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1272, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,224 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297214) roll requested 2024-12-09T08:18:17,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297224 2024-12-09T08:18:17,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297214 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297224 2024-12-09T08:18:17,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,233 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,233 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1273, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,233 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297224) roll requested 2024-12-09T08:18:17,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297234 2024-12-09T08:18:17,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297224 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297234 2024-12-09T08:18:17,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,243 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,243 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1274, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,243 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297234) roll requested 2024-12-09T08:18:17,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297243 2024-12-09T08:18:17,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297234 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297243 2024-12-09T08:18:17,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,251 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,252 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1275, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,252 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297243) roll requested 2024-12-09T08:18:17,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297252 2024-12-09T08:18:17,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297243 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297252 2024-12-09T08:18:17,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,261 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,261 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1276, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,262 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297252) roll requested 2024-12-09T08:18:17,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297262 2024-12-09T08:18:17,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297252 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297262 2024-12-09T08:18:17,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,271 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,271 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1277, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,271 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297262) roll requested 2024-12-09T08:18:17,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297271 2024-12-09T08:18:17,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297262 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297271 2024-12-09T08:18:17,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,279 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,279 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1278, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,279 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297271) roll requested 2024-12-09T08:18:17,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297279 2024-12-09T08:18:17,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297271 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297279 2024-12-09T08:18:17,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,287 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,287 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1279, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,287 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297279) roll requested 2024-12-09T08:18:17,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297288 2024-12-09T08:18:17,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297279 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297288 2024-12-09T08:18:17,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,296 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,296 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1280, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,296 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297288) roll requested 2024-12-09T08:18:17,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297297 2024-12-09T08:18:17,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297288 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297297 2024-12-09T08:18:17,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,306 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,306 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1281, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,306 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297297) roll requested 2024-12-09T08:18:17,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297306 2024-12-09T08:18:17,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297297 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297306 2024-12-09T08:18:17,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,316 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,317 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1282, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,317 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297306) roll requested 2024-12-09T08:18:17,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297317 2024-12-09T08:18:17,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297306 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297317 2024-12-09T08:18:17,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,327 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,327 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1283, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,327 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297317) roll requested 2024-12-09T08:18:17,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297327 2024-12-09T08:18:17,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297317 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297327 2024-12-09T08:18:17,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,337 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,337 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1284, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,337 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297327) roll requested 2024-12-09T08:18:17,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297338 2024-12-09T08:18:17,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297327 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297338 2024-12-09T08:18:17,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,347 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,347 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1285, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,348 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297338) roll requested 2024-12-09T08:18:17,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297348 2024-12-09T08:18:17,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297338 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297348 2024-12-09T08:18:17,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,356 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,357 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1286, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,357 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297348) roll requested 2024-12-09T08:18:17,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297357 2024-12-09T08:18:17,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297348 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297357 2024-12-09T08:18:17,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,363 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-09T08:18:17,363 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1287, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,363 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733732297357) roll requested 2024-12-09T08:18:17,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732297363 2024-12-09T08:18:17,370 ERROR [Time-limited test {}] wal.AbstractFSWAL(939): Failed log archiving for the log /a/b, java.io.IOException: throw archival exception at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.archiveLogFile(TestFailedAppendAndSync.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.archive(AbstractFSWAL.java:934) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.archive(TestFailedAppendAndSync.java:143) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync.testLockupAroundBadAssignSync(TestFailedAppendAndSync.java:280) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:18:17,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297357 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733732297363 2024-12-09T08:18:17,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-09T08:18:17,372 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9272fedbd8f9b1f79bbb491a798c861c, disabling compactions & flushes 2024-12-09T08:18:17,372 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testLockupAroundBadAssignSync,,1733732267074.9272fedbd8f9b1f79bbb491a798c861c. 2024-12-09T08:18:17,373 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1288, max=55; forcing (partial) flush of 1 region(s): 9272fedbd8f9b1f79bbb491a798c861c[MyCF] 2024-12-09T08:18:17,373 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testLockupAroundBadAssignSync,,1733732267074.9272fedbd8f9b1f79bbb491a798c861c. 2024-12-09T08:18:17,373 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 9272fedbd8f9b1f79bbb491a798c861c, because it is not online on us 2024-12-09T08:18:17,373 INFO [LogRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T08:18:17,373 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testLockupAroundBadAssignSync,,1733732267074.9272fedbd8f9b1f79bbb491a798c861c. after waiting 0 ms 2024-12-09T08:18:17,373 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testLockupAroundBadAssignSync,,1733732267074.9272fedbd8f9b1f79bbb491a798c861c. 2024-12-09T08:18:17,375 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 83 in region testLockupAroundBadAssignSync,,1733732267074.9272fedbd8f9b1f79bbb491a798c861c. 2024-12-09T08:18:17,375 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testLockupAroundBadAssignSync,,1733732267074.9272fedbd8f9b1f79bbb491a798c861c. 2024-12-09T08:18:17,375 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9272fedbd8f9b1f79bbb491a798c861c: Waiting for close lock at 1733732297372Disabling compacts and flushes for region at 1733732297372Disabling writes for close at 1733732297373 (+1 ms)Writing region close event to WAL at 1733732297375 (+2 ms)Closed at 1733732297375 2024-12-09T08:18:17,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:18:17,686 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: DodgyFSLog wal:(num 1733732297363) 2024-12-09T08:18:17,687 INFO [Time-limited test {}] regionserver.TestFailedAppendAndSync(105): Cleaning test directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e42cabe8-2a21-6cc1-fa50-a6eca4ddefd0 2024-12-09T08:18:18,225 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.TestFailedAppendAndSync#testLockupAroundBadAssignSync Thread=22 (was 11) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: process reaper java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) - Thread LEAK? -, OpenFileDescriptor=287 (was 287), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=280 (was 353), ProcessCount=11 (was 11), AvailableMemoryMB=4961 (was 5674)